<!DOCTYPE html>
<html lang="en-US" dir="ltr">
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1">
    <title>安装、启动、配置 | VitePress</title>
    <meta name="description" content="A VitePress site">
    <link rel="preload stylesheet" href="/notebook/assets/style.3dbfd0c2.css" as="style">
    
    <script type="module" src="/notebook/assets/app.8aaa4cbe.js"></script>
    <link rel="preload" href="/notebook/assets/inter-roman-latin.2ed14f66.woff2" as="font" type="font/woff2" crossorigin="">
    <link rel="modulepreload" href="/notebook/assets/chunks/framework.1336c4e5.js">
    <link rel="modulepreload" href="/notebook/assets/chunks/theme.20cddc0c.js">
    <link rel="modulepreload" href="/notebook/assets/可视化 _ 监控_监控基础.md.ac56ce4d.lean.js">
    <script id="check-dark-light">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
  </head>
  <body>
    <div id="app"><div class="Layout" data-v-255ec12d><!--[--><!--]--><!--[--><span tabindex="-1" data-v-ae3e3f51></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-ae3e3f51> Skip to content </a><!--]--><!----><header class="VPNav" data-v-255ec12d data-v-7e5bc4a5><div class="VPNavBar has-sidebar" data-v-7e5bc4a5 data-v-0937f67c><div class="container" data-v-0937f67c><div class="title" data-v-0937f67c><div class="VPNavBarTitle has-sidebar" data-v-0937f67c data-v-86d1bed8><a class="title" href="/notebook/" data-v-86d1bed8><!--[--><!--]--><!--[--><img class="VPImage logo" src="/notebook/Vue.png" alt data-v-8426fc1a><!--]--><!--[-->任硕的文档<!--]--><!--[--><!--]--></a></div></div><div class="content" data-v-0937f67c><div class="curtain" data-v-0937f67c></div><div class="content-body" data-v-0937f67c><!--[--><!--]--><div class="VPNavBarSearch search" style="--vp-meta-key:&#39;Meta&#39;;" data-v-0937f67c><!--[--><!----><div id="docsearch"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg class="DocSearch-Search-Icon" width="20" height="20" viewBox="0 0 20 20" aria-label="search icon"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-0937f67c data-v-7f418b0f><span id="main-nav-aria-label" class="visually-hidden" data-v-7f418b0f>Main Navigation</span><!--[--><!--[--><div class="VPFlyout VPNavBarMenuGroup" data-v-7f418b0f data-v-a7b5672a><button type="button" class="button" aria-haspopup="true" aria-expanded="false" data-v-a7b5672a><span class="text" data-v-a7b5672a><!----><span data-v-a7b5672a>Java学前端</span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="text-icon" data-v-a7b5672a><path d="M12,16c-0.3,0-0.5-0.1-0.7-0.3l-6-6c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l5.3,5.3l5.3-5.3c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-6,6C12.5,15.9,12.3,16,12,16z"></path></svg></span></button><div class="menu" data-v-a7b5672a><div class="VPMenu" data-v-a7b5672a data-v-e7ea1737><div class="items" data-v-e7ea1737><!--[--><!--[--><div class="VPMenuGroup" data-v-e7ea1737 data-v-69e747b5><!----><!--[--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/Java%E5%AD%A6%E5%89%8D%E7%AB%AF/HTML+JS.html" data-v-2f2cfafc><!--[-->HTML+JS<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/Java%E5%AD%A6%E5%89%8D%E7%AB%AF/CSS.html" data-v-2f2cfafc><!--[-->CSS<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/Java%E5%AD%A6%E5%89%8D%E7%AB%AF/Vue2+%E7%BB%84%E4%BB%B6.html" data-v-2f2cfafc><!--[-->Vue2+组件<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/Java%E5%AD%A6%E5%89%8D%E7%AB%AF/Vue3+%E7%BB%84%E4%BB%B6.html" data-v-2f2cfafc><!--[-->Vue3+组件<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/Java%E5%AD%A6%E5%89%8D%E7%AB%AF/React.html" data-v-2f2cfafc><!--[-->React<!--]--></a></div><!--]--><!--]--></div><!--]--><!--]--></div><!--[--><!--]--></div></div></div><!--]--><!--[--><div class="VPFlyout VPNavBarMenuGroup" data-v-7f418b0f data-v-a7b5672a><button type="button" class="button" aria-haspopup="true" aria-expanded="false" data-v-a7b5672a><span class="text" data-v-a7b5672a><!----><span data-v-a7b5672a>软件测试</span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="text-icon" data-v-a7b5672a><path d="M12,16c-0.3,0-0.5-0.1-0.7-0.3l-6-6c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l5.3,5.3l5.3-5.3c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-6,6C12.5,15.9,12.3,16,12,16z"></path></svg></span></button><div class="menu" data-v-a7b5672a><div class="VPMenu" data-v-a7b5672a data-v-e7ea1737><div class="items" data-v-e7ea1737><!--[--><!--[--><div class="VPMenuGroup" data-v-e7ea1737 data-v-69e747b5><!----><!--[--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E8%BD%AF%E4%BB%B6%E6%B5%8B%E8%AF%95/%E6%B5%8B%E8%AF%95%E5%9F%BA%E7%A1%80.html" data-v-2f2cfafc><!--[-->测试基础<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E8%BD%AF%E4%BB%B6%E6%B5%8B%E8%AF%95/%E5%8E%8B%E5%8A%9B%E6%B5%8B%E8%AF%95.html" data-v-2f2cfafc><!--[-->压力测试<!--]--></a></div><!--]--><!--]--></div><!--]--><!--]--></div><!--[--><!--]--></div></div></div><!--]--><!--[--><div class="VPFlyout VPNavBarMenuGroup" data-v-7f418b0f data-v-a7b5672a><button type="button" class="button" aria-haspopup="true" aria-expanded="false" data-v-a7b5672a><span class="text" data-v-a7b5672a><!----><span data-v-a7b5672a>多线程</span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="text-icon" data-v-a7b5672a><path d="M12,16c-0.3,0-0.5-0.1-0.7-0.3l-6-6c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l5.3,5.3l5.3-5.3c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-6,6C12.5,15.9,12.3,16,12,16z"></path></svg></span></button><div class="menu" data-v-a7b5672a><div class="VPMenu" data-v-a7b5672a data-v-e7ea1737><div class="items" data-v-e7ea1737><!--[--><!--[--><div class="VPMenuGroup" data-v-e7ea1737 data-v-69e747b5><!----><!--[--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E5%B9%B6%E5%8F%91%20&amp;%20%E5%A4%9A%E7%BA%BF%E7%A8%8B/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-2f2cfafc><!--[-->基础篇<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E5%B9%B6%E5%8F%91%20&amp;%20%E5%A4%9A%E7%BA%BF%E7%A8%8B/%E5%B9%B6%E5%8F%91%E5%AE%8C%E5%96%84.html" data-v-2f2cfafc><!--[-->进阶篇<!--]--></a></div><!--]--><!--]--></div><!--]--><!--]--></div><!--[--><!--]--></div></div></div><!--]--><!--[--><div class="VPFlyout VPNavBarMenuGroup" data-v-7f418b0f data-v-a7b5672a><button type="button" class="button" aria-haspopup="true" aria-expanded="false" data-v-a7b5672a><span class="text" data-v-a7b5672a><!----><span data-v-a7b5672a>开发工具</span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="text-icon" data-v-a7b5672a><path d="M12,16c-0.3,0-0.5-0.1-0.7-0.3l-6-6c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l5.3,5.3l5.3-5.3c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-6,6C12.5,15.9,12.3,16,12,16z"></path></svg></span></button><div class="menu" data-v-a7b5672a><div class="VPMenu" data-v-a7b5672a data-v-e7ea1737><div class="items" data-v-e7ea1737><!--[--><!--[--><div class="VPMenuGroup" data-v-e7ea1737 data-v-69e747b5><!----><!--[--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/IDEA/Chrome.html" data-v-2f2cfafc><!--[-->Chrome<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/IDEA/IDEA%E5%9F%BA%E7%A1%80.html" data-v-2f2cfafc><!--[-->IDEA基础<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/IDEA/IDEA%E6%8F%92%E4%BB%B6.html" data-v-2f2cfafc><!--[-->IDEA插件<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/IDEA/VS%20Code.html" data-v-2f2cfafc><!--[-->VS Code<!--]--></a></div><!--]--><!--]--></div><!--]--><!--]--></div><!--[--><!--]--></div></div></div><!--]--><!--[--><div class="VPFlyout VPNavBarMenuGroup" data-v-7f418b0f data-v-a7b5672a><button type="button" class="button" aria-haspopup="true" aria-expanded="false" data-v-a7b5672a><span class="text" data-v-a7b5672a><!----><span data-v-a7b5672a>消息中间件</span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="text-icon" data-v-a7b5672a><path d="M12,16c-0.3,0-0.5-0.1-0.7-0.3l-6-6c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l5.3,5.3l5.3-5.3c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-6,6C12.5,15.9,12.3,16,12,16z"></path></svg></span></button><div class="menu" data-v-a7b5672a><div class="VPMenu" data-v-a7b5672a data-v-e7ea1737><div class="items" data-v-e7ea1737><!--[--><!--[--><div class="VPMenuGroup" data-v-e7ea1737 data-v-69e747b5><!----><!--[--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6/RabbitMQ.html" data-v-2f2cfafc><!--[-->RabbitMQ<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6/RocketMQ.html" data-v-2f2cfafc><!--[-->RocketMQ<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6/Kafka.html" data-v-2f2cfafc><!--[-->Kafka<!--]--></a></div><!--]--><!--[--><div class="VPMenuLink" data-v-69e747b5 data-v-2f2cfafc><a class="VPLink link" href="/notebook/%E6%B6%88%E6%81%AF%E4%B8%AD%E9%97%B4%E4%BB%B6/Canal.html" data-v-2f2cfafc><!--[-->Canal<!--]--></a></div><!--]--><!--]--></div><!--]--><!--]--></div><!--[--><!--]--></div></div></div><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-0937f67c data-v-f6a63727><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title="toggle dark mode" aria-checked="false" data-v-f6a63727 data-v-82b282f1 data-v-f3c41672><span class="check" data-v-f3c41672><span class="icon" data-v-f3c41672><!--[--><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="sun" data-v-82b282f1><path d="M12,18c-3.3,0-6-2.7-6-6s2.7-6,6-6s6,2.7,6,6S15.3,18,12,18zM12,8c-2.2,0-4,1.8-4,4c0,2.2,1.8,4,4,4c2.2,0,4-1.8,4-4C16,9.8,14.2,8,12,8z"></path><path d="M12,4c-0.6,0-1-0.4-1-1V1c0-0.6,0.4-1,1-1s1,0.4,1,1v2C13,3.6,12.6,4,12,4z"></path><path d="M12,24c-0.6,0-1-0.4-1-1v-2c0-0.6,0.4-1,1-1s1,0.4,1,1v2C13,23.6,12.6,24,12,24z"></path><path d="M5.6,6.6c-0.3,0-0.5-0.1-0.7-0.3L3.5,4.9c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l1.4,1.4c0.4,0.4,0.4,1,0,1.4C6.2,6.5,5.9,6.6,5.6,6.6z"></path><path d="M19.8,20.8c-0.3,0-0.5-0.1-0.7-0.3l-1.4-1.4c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l1.4,1.4c0.4,0.4,0.4,1,0,1.4C20.3,20.7,20,20.8,19.8,20.8z"></path><path d="M3,13H1c-0.6,0-1-0.4-1-1s0.4-1,1-1h2c0.6,0,1,0.4,1,1S3.6,13,3,13z"></path><path d="M23,13h-2c-0.6,0-1-0.4-1-1s0.4-1,1-1h2c0.6,0,1,0.4,1,1S23.6,13,23,13z"></path><path d="M4.2,20.8c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l1.4-1.4c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-1.4,1.4C4.7,20.7,4.5,20.8,4.2,20.8z"></path><path d="M18.4,6.6c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l1.4-1.4c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-1.4,1.4C18.9,6.5,18.6,6.6,18.4,6.6z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="moon" data-v-82b282f1><path d="M12.1,22c-0.3,0-0.6,0-0.9,0c-5.5-0.5-9.5-5.4-9-10.9c0.4-4.8,4.2-8.6,9-9c0.4,0,0.8,0.2,1,0.5c0.2,0.3,0.2,0.8-0.1,1.1c-2,2.7-1.4,6.4,1.3,8.4c2.1,1.6,5,1.6,7.1,0c0.3-0.2,0.7-0.3,1.1-0.1c0.3,0.2,0.5,0.6,0.5,1c-0.2,2.7-1.5,5.1-3.6,6.8C16.6,21.2,14.4,22,12.1,22zM9.3,4.4c-2.9,1-5,3.6-5.2,6.8c-0.4,4.4,2.8,8.3,7.2,8.7c2.1,0.2,4.2-0.4,5.8-1.8c1.1-0.9,1.9-2.1,2.4-3.4c-2.5,0.9-5.3,0.5-7.5-1.1C9.2,11.4,8.1,7.7,9.3,4.4z"></path></svg><!--]--></span></span></button></div><div class="VPSocialLinks VPNavBarSocialLinks social-links" data-v-0937f67c data-v-0394ad82 data-v-7bc22406><!--[--><a class="VPSocialLink no-icon" href="https://github.com/renshuo123/renshuo123.github.io" aria-label="github" target="_blank" rel="noopener" data-v-7bc22406 data-v-f80f8133><svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>GitHub</title><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg></a><a class="VPSocialLink no-icon" href="#" aria-label="twitter" target="_blank" rel="noopener" data-v-7bc22406 data-v-f80f8133><svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Twitter</title><path d="M23.953 4.57a10 10 0 01-2.825.775 4.958 4.958 0 002.163-2.723c-.951.555-2.005.959-3.127 1.184a4.92 4.92 0 00-8.384 4.482C7.69 8.095 4.067 6.13 1.64 3.162a4.822 4.822 0 00-.666 2.475c0 1.71.87 3.213 2.188 4.096a4.904 4.904 0 01-2.228-.616v.06a4.923 4.923 0 003.946 4.827 4.996 4.996 0 01-2.212.085 4.936 4.936 0 004.604 3.417 9.867 9.867 0 01-6.102 2.105c-.39 0-.779-.023-1.17-.067a13.995 13.995 0 007.557 2.209c9.053 0 13.998-7.496 13.998-13.985 0-.21 0-.42-.015-.63A9.935 9.935 0 0024 4.59z"/></svg></a><a class="VPSocialLink no-icon" href="https://github.com/" aria-label target="_blank" rel="noopener" data-v-7bc22406 data-v-f80f8133><svg t="1676028692954" class="icon" ...</path></svg></a><!--]--></div><div class="VPFlyout VPNavBarExtra extra" data-v-0937f67c data-v-40855f84 data-v-a7b5672a><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-a7b5672a><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="icon" data-v-a7b5672a><circle cx="12" cy="12" r="2"></circle><circle cx="19" cy="12" r="2"></circle><circle cx="5" cy="12" r="2"></circle></svg></button><div class="menu" data-v-a7b5672a><div class="VPMenu" data-v-a7b5672a data-v-e7ea1737><!----><!--[--><!--[--><!----><div class="group" data-v-40855f84><div class="item appearance" data-v-40855f84><p class="label" data-v-40855f84>Appearance</p><div class="appearance-action" data-v-40855f84><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title="toggle dark mode" aria-checked="false" data-v-40855f84 data-v-82b282f1 data-v-f3c41672><span class="check" data-v-f3c41672><span class="icon" data-v-f3c41672><!--[--><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="sun" data-v-82b282f1><path d="M12,18c-3.3,0-6-2.7-6-6s2.7-6,6-6s6,2.7,6,6S15.3,18,12,18zM12,8c-2.2,0-4,1.8-4,4c0,2.2,1.8,4,4,4c2.2,0,4-1.8,4-4C16,9.8,14.2,8,12,8z"></path><path d="M12,4c-0.6,0-1-0.4-1-1V1c0-0.6,0.4-1,1-1s1,0.4,1,1v2C13,3.6,12.6,4,12,4z"></path><path d="M12,24c-0.6,0-1-0.4-1-1v-2c0-0.6,0.4-1,1-1s1,0.4,1,1v2C13,23.6,12.6,24,12,24z"></path><path d="M5.6,6.6c-0.3,0-0.5-0.1-0.7-0.3L3.5,4.9c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l1.4,1.4c0.4,0.4,0.4,1,0,1.4C6.2,6.5,5.9,6.6,5.6,6.6z"></path><path d="M19.8,20.8c-0.3,0-0.5-0.1-0.7-0.3l-1.4-1.4c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l1.4,1.4c0.4,0.4,0.4,1,0,1.4C20.3,20.7,20,20.8,19.8,20.8z"></path><path d="M3,13H1c-0.6,0-1-0.4-1-1s0.4-1,1-1h2c0.6,0,1,0.4,1,1S3.6,13,3,13z"></path><path d="M23,13h-2c-0.6,0-1-0.4-1-1s0.4-1,1-1h2c0.6,0,1,0.4,1,1S23.6,13,23,13z"></path><path d="M4.2,20.8c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l1.4-1.4c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-1.4,1.4C4.7,20.7,4.5,20.8,4.2,20.8z"></path><path d="M18.4,6.6c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l1.4-1.4c0.4-0.4,1-0.4,1.4,0s0.4,1,0,1.4l-1.4,1.4C18.9,6.5,18.6,6.6,18.4,6.6z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="moon" data-v-82b282f1><path d="M12.1,22c-0.3,0-0.6,0-0.9,0c-5.5-0.5-9.5-5.4-9-10.9c0.4-4.8,4.2-8.6,9-9c0.4,0,0.8,0.2,1,0.5c0.2,0.3,0.2,0.8-0.1,1.1c-2,2.7-1.4,6.4,1.3,8.4c2.1,1.6,5,1.6,7.1,0c0.3-0.2,0.7-0.3,1.1-0.1c0.3,0.2,0.5,0.6,0.5,1c-0.2,2.7-1.5,5.1-3.6,6.8C16.6,21.2,14.4,22,12.1,22zM9.3,4.4c-2.9,1-5,3.6-5.2,6.8c-0.4,4.4,2.8,8.3,7.2,8.7c2.1,0.2,4.2-0.4,5.8-1.8c1.1-0.9,1.9-2.1,2.4-3.4c-2.5,0.9-5.3,0.5-7.5-1.1C9.2,11.4,8.1,7.7,9.3,4.4z"></path></svg><!--]--></span></span></button></div></div></div><div class="group" data-v-40855f84><div class="item social-links" data-v-40855f84><div class="VPSocialLinks social-links-list" data-v-40855f84 data-v-7bc22406><!--[--><a class="VPSocialLink no-icon" href="https://github.com/renshuo123/renshuo123.github.io" aria-label="github" target="_blank" rel="noopener" data-v-7bc22406 data-v-f80f8133><svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>GitHub</title><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg></a><a class="VPSocialLink no-icon" href="#" aria-label="twitter" target="_blank" rel="noopener" data-v-7bc22406 data-v-f80f8133><svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Twitter</title><path d="M23.953 4.57a10 10 0 01-2.825.775 4.958 4.958 0 002.163-2.723c-.951.555-2.005.959-3.127 1.184a4.92 4.92 0 00-8.384 4.482C7.69 8.095 4.067 6.13 1.64 3.162a4.822 4.822 0 00-.666 2.475c0 1.71.87 3.213 2.188 4.096a4.904 4.904 0 01-2.228-.616v.06a4.923 4.923 0 003.946 4.827 4.996 4.996 0 01-2.212.085 4.936 4.936 0 004.604 3.417 9.867 9.867 0 01-6.102 2.105c-.39 0-.779-.023-1.17-.067a13.995 13.995 0 007.557 2.209c9.053 0 13.998-7.496 13.998-13.985 0-.21 0-.42-.015-.63A9.935 9.935 0 0024 4.59z"/></svg></a><a class="VPSocialLink no-icon" href="https://github.com/" aria-label target="_blank" rel="noopener" data-v-7bc22406 data-v-f80f8133><svg t="1676028692954" class="icon" ...</path></svg></a><!--]--></div></div></div><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-0937f67c data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><!----></header><div class="VPLocalNav reached-top" data-v-255ec12d data-v-5cfd5582><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-5cfd5582><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="menu-icon" data-v-5cfd5582><path d="M17,11H3c-0.6,0-1-0.4-1-1s0.4-1,1-1h14c0.6,0,1,0.4,1,1S17.6,11,17,11z"></path><path d="M21,7H3C2.4,7,2,6.6,2,6s0.4-1,1-1h18c0.6,0,1,0.4,1,1S21.6,7,21,7z"></path><path d="M21,15H3c-0.6,0-1-0.4-1-1s0.4-1,1-1h18c0.6,0,1,0.4,1,1S21.6,15,21,15z"></path><path d="M17,19H3c-0.6,0-1-0.4-1-1s0.4-1,1-1h14c0.6,0,1,0.4,1,1S17.6,19,17,19z"></path></svg><span class="menu-text" data-v-5cfd5582>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-5cfd5582 data-v-18201f51><button data-v-18201f51>Return to top</button><!----></div></div><aside class="VPSidebar" data-v-255ec12d data-v-845b8fc6><div class="curtain" data-v-845b8fc6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-845b8fc6><span class="visually-hidden" id="sidebar-aria-label" data-v-845b8fc6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>Java</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Java/Java%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Java基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Java/Java%E6%96%B0%E7%89%B9%E6%80%A7.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Java新特性</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Java/Java%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Java进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Java/Java%E9%9B%86%E5%90%88.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Java集合</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Java/Java%E9%AB%98%E7%BA%A7.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Java高级</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>Linux</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Linux/Linux%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Linux基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Linux/Linux%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Linux新特性</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Linux/Shell.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Shell脚本</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Linux/%E5%AE%9E%E7%94%A8%E8%84%9A%E6%9C%AC.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>实用脚本</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Linux/%E8%BD%AF%E4%BB%B6%E9%83%A8%E7%BD%B2.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>软件部署</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>Nginx</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Nginx/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>基础篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Nginx/%E8%BF%9B%E9%98%B6%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Nginx/%E5%AE%9E%E6%88%98%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>实战篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Nginx/%E9%9D%A2%E8%AF%95%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>面试篇</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>SSM</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/SSM/Maven.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Maven</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/SSM/Spring.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Spring</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/SSM/SpringMVC.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>SpringMVC</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/SSM/SpringBatch.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>SpringBatch</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>SpringBoot</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/3%E3%80%81SpringBoot/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>基础篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/3%E3%80%81SpringBoot/%E5%BA%94%E7%94%A8%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>应用篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/3%E3%80%81SpringBoot/%E6%96%B0%E7%89%B9%E6%80%A7.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>新特性</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/3%E3%80%81SpringBoot/%E8%BF%90%E7%BB%B4&amp;%E5%8E%9F%E7%90%86.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>运维&原理</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>SpringCloud</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/4%E3%80%81%E5%BE%AE%E6%9C%8D%E5%8A%A1/%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>SpringCloud</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/4%E3%80%81%E5%BE%AE%E6%9C%8D%E5%8A%A1/%E5%BF%85%E5%A4%87/Sentinel.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Sentinel</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>SpringSecurity</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/4%E3%80%81%E5%BE%AE%E6%9C%8D%E5%8A%A1/SpringSecurity/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>SpringSecurity基础篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/4%E3%80%81%E5%BE%AE%E6%9C%8D%E5%8A%A1/SpringSecurity/%E8%BF%9B%E9%98%B6%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>SpringSecurity进阶篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/4%E3%80%81%E5%BE%AE%E6%9C%8D%E5%8A%A1/SpringSecurity/%E9%AB%98%E7%BA%A7%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>SpringSecurity高级篇</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>Mybatis & MybatisPlus</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Mybatis&amp;MybatisPlus/Mybatis.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Mybatis</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Mybatis&amp;MybatisPlus/MybatisPlus.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MybatisPlus</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/Mybatis&amp;MybatisPlus/JPA.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>JPA</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>Git & ChatGPT</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/5%E3%80%81%E8%BF%90%E7%BB%B4/Git.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Git</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/5%E3%80%81%E8%BF%90%E7%BB%B4/Github.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Github</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/5%E3%80%81%E8%BF%90%E7%BB%B4/ChatGPT.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>ChatGPT</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/5%E3%80%81%E8%BF%90%E7%BB%B4/Jenkins.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Jenkins</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/5%E3%80%81%E8%BF%90%E7%BB%B4/Netty.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Netty</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>数据库</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>MySQL</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MySQL/MySQL%E6%A0%B8%E5%BF%83/%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MySQL基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MySQL/MySQL%E6%A0%B8%E5%BF%83/%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MySQL进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MySQL/MySQL%E6%A0%B8%E5%BF%83/%E4%BC%98%E5%8C%96.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MySQL优化</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MySQL/MySQL%E6%A0%B8%E5%BF%83/%E8%AE%BE%E8%AE%A1.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MySQL设计</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MySQL/MySQL%E6%A0%B8%E5%BF%83/%E8%BF%90%E7%BB%B4.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MySQL运维</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MySQL/%E5%88%86%E5%BA%93%E5%88%86%E8%A1%A8.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>分库分表</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>Redis</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Redis/Redis%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Redis基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Redis/Redis%E4%BC%98%E5%8C%96.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Redis优化</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Redis/Redis%E5%8E%9F%E7%90%86.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Redis原理</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Redis/Redis%E9%AB%98%E7%BA%A7.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Redis高级</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Redis/Redis%E5%AE%9E%E6%88%98.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Redis实战</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Redis/%E6%9C%AC%E5%9C%B0%E7%BC%93%E5%AD%98.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>本地缓存</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>MongoDB</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MongoDB/%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MongoDB基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/MongoDB/%E6%95%B4%E5%90%88.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>MongoDB进阶</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>ElasticSearch</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/ElasticSearch/1%E3%80%81ES%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>ES基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/ElasticSearch/3%E3%80%81ES%E9%AB%98%E7%BA%A7.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>ES高级</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/influxdb.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>InfluxDB</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/2%E3%80%81%E6%95%B0%E6%8D%AE%E5%BA%93/Neo4j.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Neo4j</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>高并发 & 秒杀 & 分布式</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E4%B8%89%E9%AB%98/%E5%88%86%E5%B8%83%E5%BC%8F.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>分布式理论</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/4%E3%80%81%E5%BE%AE%E6%9C%8D%E5%8A%A1/%E5%BF%85%E5%A4%87/%E5%88%86%E5%B8%83%E5%BC%8F%E9%94%81.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>分布式锁</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E4%B8%89%E9%AB%98/%E7%A7%92%E6%9D%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>秒杀</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E4%B8%89%E9%AB%98/%E9%AB%98%E5%8F%AF%E7%94%A8.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>高可用</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E4%B8%89%E9%AB%98/%E9%AB%98%E5%B9%B6%E5%8F%91.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>高并发</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>云原生</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E4%BA%91%E5%8E%9F%E7%94%9F/Docker.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Docker</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E4%BA%91%E5%8E%9F%E7%94%9F/K8S.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>K8S</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible has-active" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>可视化 & 监控</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E5%8F%AF%E8%A7%86%E5%8C%96%20&amp;%20%E7%9B%91%E6%8E%A7/%E7%9B%91%E6%8E%A7%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>监控基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E5%8F%AF%E8%A7%86%E5%8C%96%20&amp;%20%E7%9B%91%E6%8E%A7/%E7%9B%91%E6%8E%A7%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>监控进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E5%8F%AF%E8%A7%86%E5%8C%96%20&amp;%20%E7%9B%91%E6%8E%A7/%E5%8F%AF%E8%A7%86%E5%8C%96%E5%A4%A7%E5%B1%8F.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>可视化大屏</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E5%8F%AF%E8%A7%86%E5%8C%96%20&amp;%20%E7%9B%91%E6%8E%A7/Zabbix.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Zabbix</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>学前端</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>HTML+CSS</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/1%E3%80%81HTML+CSS/HTML%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>HTML基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/1%E3%80%81HTML+CSS/CSS%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>CSS基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/1%E3%80%81HTML+CSS/%E7%BD%91%E9%A1%B5%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>网页进阶</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>JS+TS</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/2%E3%80%81JS+TS/JS%20%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>JS基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/2%E3%80%81JS+TS/JS%20%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>JS进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/2%E3%80%81JS+TS/ES6%20%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>ES6基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/2%E3%80%81JS+TS/ES6%20%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>ES6进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/2%E3%80%81JS+TS/TypeScript.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>TS基础</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>NodeJS</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/4%E3%80%81Node/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Node基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/4%E3%80%81Node/%E8%BF%9B%E9%98%B6%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Node进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/4%E3%80%81Node/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>项目实战</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>Vue</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/3%E3%80%81Vue/Vue3/Vue3%E8%BF%9B%E9%98%B6.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Vue3进阶</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/3%E3%80%81Vue/Vue3/Vue3%E9%AB%98%E7%BA%A7.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Vue3高级</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/3%E3%80%81Vue/Vue3/Vue3%E6%96%B0%E8%AF%AD%E6%B3%95.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>Vue3新语法</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/3%E3%80%81Vue/Vue2/Vue2%E9%A1%B9%E7%9B%AE.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>项目实战</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>小程序</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/5%E3%80%81%E5%B0%8F%E7%A8%8B%E5%BA%8F/%E5%BE%AE%E4%BF%A1%E5%B0%8F%E7%A8%8B%E5%BA%8F.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>小程序基础</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/5%E3%80%81%E5%B0%8F%E7%A8%8B%E5%BA%8F/%E5%B0%8F%E7%A8%8B%E5%BA%8F%E4%BC%98%E5%8C%96.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>小程序优化</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/5%E3%80%81%E5%B0%8F%E7%A8%8B%E5%BA%8F/uniapp.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>uniapp</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/1%E3%80%81%E5%AD%A6%E5%89%8D%E7%AB%AF/5%E3%80%81%E5%B0%8F%E7%A8%8B%E5%BA%8F/%E5%B0%8F%E7%A8%8B%E5%BA%8F%E9%A1%B9%E7%9B%AE.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>项目实战</p><!--]--></a><!----></div><!----></div><!--]--></div></section><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>计算机基础</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>数据结构</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E6%93%8D%E4%BD%9C%E7%B3%BB%E7%BB%9F.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>操作系统</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E8%AE%BE%E8%AE%A1%E6%A8%A1%E5%BC%8F/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>设计模式</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E8%AE%A1%E7%AE%97%E6%9C%BA%E7%BD%91%E7%BB%9C/%E7%BD%91%E7%BB%9C%E5%9F%BA%E7%A1%80.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>计算机网络</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E8%AE%BE%E8%AE%A1%E6%A8%A1%E5%BC%8F/UML.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>UML</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E8%AE%A1%E7%AE%97%E6%9C%BA%E5%9F%BA%E7%A1%80/%E7%AE%97%E6%B3%95/LeetCode.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>LeetCode</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0 collapsible collapsed" data-v-845b8fc6 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h2 class="text" data-v-9b797284>项目实战</h2><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>云尚办公</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E4%BA%91%E5%B0%9A%E5%8A%9E%E5%85%AC/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>基础篇</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>小兔鲜</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E5%B0%8F%E5%85%94%E9%B2%9C/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>基础篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E5%B0%8F%E5%85%94%E9%B2%9C/%E8%BF%9B%E9%98%B6%E7%AF%871.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇1</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E5%B0%8F%E5%85%94%E9%B2%9C/%E8%BF%9B%E9%98%B6%E7%AF%872.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇2</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>地图</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E7%99%BE%E5%BA%A6%E5%9C%B0%E5%9B%BE/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>基础篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E7%99%BE%E5%BA%A6%E5%9C%B0%E5%9B%BE/%E8%BF%9B%E9%98%B6%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>苍穹外卖</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E8%8B%8D%E7%A9%B9%E5%A4%96%E5%8D%96/%E8%BF%9B%E9%98%B6%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-1 collapsible collapsed" data-v-9b797284 data-v-9b797284><div class="item" role="button" tabindex="0" data-v-9b797284><div class="indicator" data-v-9b797284></div><h3 class="text" data-v-9b797284>黑马头条</h3><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-9b797284><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" viewbox="0 0 24 24" class="caret-icon" data-v-9b797284><path d="M9,19c-0.3,0-0.5-0.1-0.7-0.3c-0.4-0.4-0.4-1,0-1.4l5.3-5.3L8.3,6.7c-0.4-0.4-0.4-1,0-1.4s1-0.4,1.4,0l6,6c0.4,0.4,0.4,1,0,1.4l-6,6C9.5,18.9,9.3,19,9,19z"></path></svg></div></div><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E9%BB%91%E9%A9%AC%E5%A4%B4%E6%9D%A1/%E5%9F%BA%E7%A1%80%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>基础篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E9%BB%91%E9%A9%AC%E5%A4%B4%E6%9D%A1/%E8%BF%9B%E9%98%B6%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E9%BB%91%E9%A9%AC%E5%A4%B4%E6%9D%A1/%E8%BF%9B%E9%98%B6%E7%AF%872.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>进阶篇2</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E9%BB%91%E9%A9%AC%E5%A4%B4%E6%9D%A1/%E9%AB%98%E7%BA%A7%E7%AF%87.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>高级篇</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E6%94%AF%E4%BB%98.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>支付</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/%E9%A1%B9%E7%9B%AE%E5%AE%9E%E6%88%98/%E9%A1%B9%E7%9B%AE%E6%8E%A8%E8%8D%90.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>项目推荐</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="group" data-v-845b8fc6><section class="VPSidebarItem level-0" data-v-845b8fc6 data-v-9b797284><!----><div class="items" data-v-9b797284><!--[--><div class="VPSidebarItem level-1 is-link" data-v-9b797284 data-v-9b797284><div class="item" data-v-9b797284><div class="indicator" data-v-9b797284></div><a class="VPLink link link" href="/notebook/team.html" data-v-9b797284><!--[--><p class="text" data-v-9b797284>团队成员</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-255ec12d data-v-669faec9><div class="VPDoc has-sidebar has-aside" data-v-669faec9 data-v-6b87e69f><!--[--><!--]--><div class="container" data-v-6b87e69f><div class="aside" data-v-6b87e69f><div class="aside-curtain" data-v-6b87e69f></div><div class="aside-container" data-v-6b87e69f><div class="aside-content" data-v-6b87e69f><div class="VPDocAside" data-v-6b87e69f data-v-3f215769><!--[--><!--]--><!--[--><!--]--><div class="VPDocAsideOutline" data-v-3f215769 data-v-ff0f39c8><div class="content" data-v-ff0f39c8><div class="outline-marker" data-v-ff0f39c8></div><div class="outline-title" data-v-ff0f39c8>On this page</div><nav aria-labelledby="doc-outline-aria-label" data-v-ff0f39c8><span class="visually-hidden" id="doc-outline-aria-label" data-v-ff0f39c8> Table of Contents for current page </span><ul class="root" data-v-ff0f39c8 data-v-d0ee3533><!--[--><!--]--></ul></nav></div></div><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-6b87e69f><div class="content-container" data-v-6b87e69f><!--[--><!--]--><!----><main class="main" data-v-6b87e69f><div style="position:relative;" class="vp-doc _notebook_%E5%8F%AF%E8%A7%86%E5%8C%96%20&amp;%20%E7%9B%91%E6%8E%A7_%E7%9B%91%E6%8E%A7%E5%9F%BA%E7%A1%80" data-v-6b87e69f><div><h1 id="安装、启动、配置" tabindex="-1">安装、启动、配置 <a class="header-anchor" href="#安装、启动、配置" aria-label="Permalink to &quot;安装、启动、配置&quot;">​</a></h1><p>官网：<a href="https://prometheus.io/" target="_blank" rel="noreferrer">https://prometheus.io/</a></p><p>下载地址：<a href="https://prometheus.io/download/" target="_blank" rel="noreferrer">https://prometheus.io/download/</a></p><h2 id="二进制版安装" tabindex="-1">二进制版安装 <a class="header-anchor" href="#二进制版安装" aria-label="Permalink to &quot;二进制版安装&quot;">​</a></h2><h3 id="最简安装" tabindex="-1">最简安装 <a class="header-anchor" href="#最简安装" aria-label="Permalink to &quot;最简安装&quot;">​</a></h3><p><a href="https://gitee.com/linge365/prometheus" target="_blank" rel="noreferrer">https://gitee.com/linge365/prometheus</a></p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt</span></span>
<span class="line"><span style="color:#FFCB6B;">git</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">clone</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://gitee.com/linge365/prometheus.git</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 移动systemd 服务到/etc/systemd/system/目录下</span></span>
<span class="line"><span style="color:#FFCB6B;">mv</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#C3E88D;">.service</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/systemd/system/</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#检查</span></span>
<span class="line"><span style="color:#FFCB6B;">ls</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-l</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/systemd/system/</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 创建一个专门的 prometheus 用户：</span></span>
<span class="line"><span style="color:#FFCB6B;">useradd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-M</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-s</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/usr/sbin/nologin</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 更改 prometheus 用户的文件夹权限：</span></span>
<span class="line"><span style="color:#FFCB6B;">chown</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus:prometheus</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-R</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/prometheus</span></span></code></pre></div><h3 id="启动命令" tabindex="-1">启动命令 <a class="header-anchor" href="#启动命令" aria-label="Permalink to &quot;启动命令&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">daemon-reload</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">grafana-server.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager.service</span></span></code></pre></div><h3 id="设置开机自启" tabindex="-1">设置开机自启 <a class="header-anchor" href="#设置开机自启" aria-label="Permalink to &quot;设置开机自启&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">enable</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">enable</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">grafana-server.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">enable</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">enable</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager.service</span></span></code></pre></div><h3 id="检查启动状态" tabindex="-1">检查启动状态 <a class="header-anchor" href="#检查启动状态" aria-label="Permalink to &quot;检查启动状态&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">status</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">status</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">status</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">grafana-server.service</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">status</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager.service</span></span>
<span class="line"><span style="color:#FFCB6B;">ss</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-lntp</span><span style="color:#89DDFF;">|</span><span style="color:#FFCB6B;">egrep</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">3000|9090|9100|9093</span><span style="color:#89DDFF;">&quot;</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011102670.png" alt="image-20230501110215582" style="zoom:80%;"><h2 id="docker版安装" tabindex="-1">Docker版安装 <a class="header-anchor" href="#docker版安装" aria-label="Permalink to &quot;Docker版安装&quot;">​</a></h2><h3 id="安装docker" tabindex="-1">安装Docker <a class="header-anchor" href="#安装docker" aria-label="Permalink to &quot;安装Docker&quot;">​</a></h3><div class="language-json"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">sudo mkdir -p /etc/docker</span></span>
<span class="line"><span style="color:#A6ACCD;">sudo tee /etc/docker/daemon.json &lt;&lt;-&#39;EOF&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">{</span></span>
<span class="line"><span style="color:#A6ACCD;">   </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C792EA;">registry-mirrors</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">http://hub-mirror.c.163.com</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#89DDFF;">}</span></span>
<span class="line"><span style="color:#A6ACCD;">EOF</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C792EA;">export</span><span style="color:#A6ACCD;"> DOWNLOAD_URL</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">http://mirrors.163.com/docker-ce</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-fsSL</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://get.docker.com/</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">|</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">sh</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-v</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">enable</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">status</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker</span></span></code></pre></div><h3 id="安装docker-compose" tabindex="-1">安装Docker-compose <a class="header-anchor" href="#安装docker-compose" aria-label="Permalink to &quot;安装Docker-compose&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># Compose目前已经完全支持Linux、Mac OS和Windows，在我们安装Compose之前，需要先安装Docker。下面我 们以编译好的二进制包方式安装在Linux系统中。 </span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-L</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">https://github.com/docker/compose/releases/download/1.28.6/docker-compose-</span><span style="color:#89DDFF;">$(</span><span style="color:#FFCB6B;">uname</span><span style="color:#C3E88D;"> -s</span><span style="color:#89DDFF;">)</span><span style="color:#C3E88D;">-</span><span style="color:#89DDFF;">$(</span><span style="color:#FFCB6B;">uname</span><span style="color:#C3E88D;"> -m</span><span style="color:#89DDFF;">)&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-o</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/usr/local/bin/docker-compose</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 设置文件可执行权限 </span></span>
<span class="line"><span style="color:#FFCB6B;">sudo</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">chmod</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">+x</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/usr/local/bin/docker-compose</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看版本信息 </span></span>
<span class="line"><span style="color:#FFCB6B;">docker-compose</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-version</span></span></code></pre></div><h3 id="查看最新版本" tabindex="-1">查看最新版本 <a class="header-anchor" href="#查看最新版本" aria-label="Permalink to &quot;查看最新版本&quot;">​</a></h3><p><a href="https://hub.docker.com/r/prom/prometheus/tags" target="_blank" rel="noreferrer">https://hub.docker.com/r/prom/prometheus/tags</a></p><p><a href="https://hub.docker.com/r/prom/alertmanager/tags" target="_blank" rel="noreferrer">https://hub.docker.com/r/prom/alertmanager/tags</a></p><p><a href="https://hub.docker.com/r/prom/node-exporter/tags" target="_blank" rel="noreferrer">https://hub.docker.com/r/prom/node-exporter/tags</a></p><h3 id="最简安装⭐" tabindex="-1">最简安装⭐ <a class="header-anchor" href="#最简安装⭐" aria-label="Permalink to &quot;最简安装⭐&quot;">​</a></h3><p><a href="https://gitee.com/linge365/docker-prometheus" target="_blank" rel="noreferrer">https://gitee.com/linge365/docker-prometheus</a></p><p>注意：此时所有内容均放在Gitee仓库中了，克隆下来即可</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/</span></span>
<span class="line"><span style="color:#FFCB6B;">yum</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">install</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-y</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">git</span></span>
<span class="line"><span style="color:#FFCB6B;">git</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">clone</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://gitee.com/linge365/docker-prometheus.git</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker-prometheus</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 如果下载没成功，就多试几次</span></span>
<span class="line"><span style="color:#FFCB6B;">docker-compose</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span></span></code></pre></div><h3 id="检查端口" tabindex="-1">检查端口 <a class="header-anchor" href="#检查端口" aria-label="Permalink to &quot;检查端口&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">ps</span></span>
<span class="line"><span style="color:#FFCB6B;">ss</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-lntp</span><span style="color:#89DDFF;">|</span><span style="color:#FFCB6B;">egrep</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">3000|9090|9100|9093</span><span style="color:#89DDFF;">&quot;</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011102670.png" alt="image-20230501110215582" style="zoom:80%;"><h3 id="图形展示" tabindex="-1">图形展示 <a class="header-anchor" href="#图形展示" aria-label="Permalink to &quot;图形展示&quot;">​</a></h3><blockquote><p>登录Grafana <a href="http://192.168.88.102:3000/" target="_blank" rel="noreferrer">http://192.168.88.102:3000/</a></p><p>用户名：admin，密码：password</p></blockquote><p>在 Grafana 中创建 Prometheus 数据源：</p><p>单击边栏中的“齿轮”，打开“配置”菜单。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011114421.png" alt="image-20230501111415294" style="zoom:67%;"><p>选择“Prometheus”作为类型。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011114900.png" alt="image-20230501111434779" style="zoom:80%;"><p>url填入： <a href="http://prometheus:9090" target="_blank" rel="noreferrer">http://prometheus:9090</a> 注：docker容器一般通过主机名链接，因为ip会变。单击“保存并测试”</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011115756.png" alt="image-20230501111509634" style="zoom:80%;"><p>导入仪表盘：<a href="https://grafana.com/grafana/dashboards/" target="_blank" rel="noreferrer">https://grafana.com/grafana/dashboards/</a></p><p>打开grafana的dashboards官网，在搜索栏输入node exporter回车，点击下载量大的dashboards如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011116933.png" alt="image-20230501111613771" style="zoom:67%;"><p>打开grafana web控制台--点击dashbord图标--在点import导入--粘贴之前复制的id, 1860 --在点load加载</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011116142.png" alt="image-20230501111634967" style="zoom:67%;"><p>name填写“服务器监控”（根据实际修改），选择“prometheus”--在点import导入</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011116949.png" alt="image-20230501111653790" style="zoom:67%;"><p>导入dashbord完成后，如下图</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011117333.png" alt="image-20230501111709195" style="zoom:80%;"><h3 id="内容详解" tabindex="-1">内容详解 <a class="header-anchor" href="#内容详解" aria-label="Permalink to &quot;内容详解&quot;">​</a></h3><blockquote><p>Prometheus 采集数据</p><p>Grafana 用于图表展示</p><p>alertmanager 用于接收 Prometheus 发送的告警信息</p><p>node-exporter 用于收集操作系统和硬件信息的metrics</p><p>cadvisor 用于收集docker的相关metrics</p></blockquote><h4 id="docker-compose-yml" tabindex="-1">docker-compose.yml <a class="header-anchor" href="#docker-compose-yml" aria-label="Permalink to &quot;docker-compose.yml&quot;">​</a></h4><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">version</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">3.3</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">prometheus_data</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{}</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">grafana_data</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{}</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">networks</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">monitoring</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">driver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">bridge</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">prometheus</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prom/prometheus:v2.37.6</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/localtime:/etc/localtime:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./prometheus/:/etc/prometheus/</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus_data:/prometheus</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--config.file=/etc/prometheus/prometheus.yml</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--storage.tsdb.path=/prometheus</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--web.console.libraries=/usr/share/prometheus/console_libraries</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--web.console.templates=/usr/share/prometheus/consoles</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">      </span><span style="color:#676E95;font-style:italic;">#热加载配置</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--web.enable-lifecycle</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">      </span><span style="color:#676E95;font-style:italic;">#api配置</span></span>
<span class="line"><span style="color:#89DDFF;">      </span><span style="color:#676E95;font-style:italic;">#- &#39;--web.enable-admin-api&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">      </span><span style="color:#676E95;font-style:italic;">#历史数据最大保留时间，默认15天</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--storage.tsdb.retention.time=30d</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">networks</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">monitoring</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">links</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">cadvisor</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">expose</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">9090</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">9090:9090</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">depends_on</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">cadvisor</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">alertmanager</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prom/alertmanager:v0.25.0</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/localtime:/etc/localtime:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./alertmanager/:/etc/alertmanager/</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--config.file=/etc/alertmanager/config.yml</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--storage.path=/alertmanager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">networks</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">monitoring</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">expose</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">9093</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">9093:9093</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">cadvisor</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">google/cadvisor:latest</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">cadvisor</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/localtime:/etc/localtime:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/:/rootfs:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/var/run:/var/run:rw</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/sys:/sys:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/var/lib/docker/:/var/lib/docker:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">networks</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">monitoring</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">expose</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">8080</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">node_exporter</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prom/node-exporter:v1.5.0</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/localtime:/etc/localtime:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/proc:/host/proc:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/sys:/host/sys:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/:/rootfs:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--path.procfs=/host/proc</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--path.sysfs=/host/sys</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc|rootfs/var/lib/docker)($$|/)</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">networks</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">monitoring</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">9100:9100</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">grafana</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">grafana/grafana:9.4.3</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">grafana</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/localtime:/etc/localtime:ro</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">grafana_data:/var/lib/grafana</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./grafana/provisioning/:/etc/grafana/provisioning/</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">env_file</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./grafana/config.monitoring</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">networks</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">monitoring</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">links</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">3000:3000</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">depends_on</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus</span></span></code></pre></div><h4 id="prometheus-alert-yml" tabindex="-1">prometheus/alert.yml <a class="header-anchor" href="#prometheus-alert-yml" aria-label="Permalink to &quot;prometheus/alert.yml&quot;">​</a></h4><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">groups</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus alert</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># 对任何实例超过30秒无法联系的情况发出警报</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">alert</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">服务告警</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">expr</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up == 0</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">for</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">30s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">critical</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">annotations</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">summary</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">服务异常,实例:{{ $labels.instance }}</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">description</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">{{ $labels.job }} 服务已关闭</span><span style="color:#89DDFF;">&quot;</span></span></code></pre></div><h4 id="prometheus-prometheus-yml" tabindex="-1">prometheus/prometheus.yml <a class="header-anchor" href="#prometheus-prometheus-yml" aria-label="Permalink to &quot;prometheus/prometheus.yml&quot;">​</a></h4><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 全局配置</span></span>
<span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;">     </span><span style="color:#C3E88D;">15s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 将搜刮间隔设置为每15秒一次。默认是每1分钟一次。</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">evaluation_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 每15秒评估一次规则。默认是每1分钟一次。</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># Alertmanager 配置</span></span>
<span class="line"><span style="color:#F07178;">alerting</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">alertmanagers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 报警(触发器)配置</span></span>
<span class="line"><span style="color:#F07178;">rule_files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">alert.yml</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 搜刮配置</span></span>
<span class="line"><span style="color:#F07178;">scrape_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">    </span><span style="color:#676E95;font-style:italic;"># 覆盖全局默认值，每15秒从该作业中刮取一次目标</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">localhost:9090</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cadvisor</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cadvisor:8080</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">node_exporter:9100</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus服务器</span></span></code></pre></div><h4 id="grafana-config-monitoring" tabindex="-1">grafana/config.monitoring <a class="header-anchor" href="#grafana-config-monitoring" aria-label="Permalink to &quot;grafana/config.monitoring&quot;">​</a></h4><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">GF_SECURITY_ADMIN_PASSWORD</span><span style="color:#89DDFF;">=</span><span style="color:#C3E88D;">password</span></span>
<span class="line"><span style="color:#A6ACCD;">GF_USERS_ALLOW_SIGN_UP</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">false</span></span></code></pre></div><h4 id="alertmanager-config-yml" tabindex="-1">alertmanager/config.yml <a class="header-anchor" href="#alertmanager-config-yml" aria-label="Permalink to &quot;alertmanager/config.yml&quot;">​</a></h4><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#163服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_smarthost</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">smtp.163.com:465</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#发邮件的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_from</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#发邮件的邮箱用户名，也就是你的邮箱　　　　　</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_auth_username</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#发邮件的邮箱密码</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_auth_password</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">your-password</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#进行tls验证</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_require_tls</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">false</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># 当收到告警的时候，等待group_wait配置的时间，看是否还有告警，如果有就一起发出去</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#  如果上次告警信息发送成功，此时又来了一个新的告警数据，则需要等待group_interval配置的时间才可以发送出去</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># 如果上次告警信息发送成功，且问题没有解决，则等待 repeat_interval配置的时间再次发送告警数据</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">repeat_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10m</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># 全局报警组，这个参数是必选的</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">email</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">email</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#收邮件的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#F07178;">inhibit_rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">     </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">critical</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">   </span><span style="color:#F07178;">target_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">     </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">warning</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">   </span><span style="color:#F07178;">equal</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dev</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">instance</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><h2 id="web-页面查看" tabindex="-1">web 页面查看 <a class="header-anchor" href="#web-页面查看" aria-label="Permalink to &quot;web 页面查看&quot;">​</a></h2><p>浏览器输入：<a href="http://node1:9090/" target="_blank" rel="noreferrer">http://node1:9090/</a></p><h3 id="查看启动情况" tabindex="-1">查看启动情况 <a class="header-anchor" href="#查看启动情况" aria-label="Permalink to &quot;查看启动情况&quot;">​</a></h3><p>点击 Status，选中 Targets：prometheus、pushgateway 和 node exporter 都是 up 状态，表示安装启动成功：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251019922.png" alt="image-20230425101903809" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251019419.png" alt="image-20230425101923356" style="zoom:80%;"><h3 id="查看规则配置" tabindex="-1">查看规则配置 <a class="header-anchor" href="#查看规则配置" aria-label="Permalink to &quot;查看规则配置&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251019511.png" alt="image-20230425101949440" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251020084.png" alt="image-20230425102004982" style="zoom:80%;"><h3 id="查看告警" tabindex="-1">查看告警 <a class="header-anchor" href="#查看告警" aria-label="Permalink to &quot;查看告警&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251017286.png" alt="image-20230425101719191" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251018295.png" alt="image-20230425101825205" style="zoom:80%;"><h2 id="动态加载配置" tabindex="-1">动态加载配置 <a class="header-anchor" href="#动态加载配置" aria-label="Permalink to &quot;动态加载配置&quot;">​</a></h2><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># prometheus</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://node1:9090/-/reload</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># altermanager</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://node1:9093/-/reload</span></span></code></pre></div><p>执行命令检查规则配置是否正确</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">./promtool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">check</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">config</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.yml</span></span>
<span class="line"><span style="color:#FFCB6B;">./promtool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">check</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">rules</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">first_rules.yml</span></span>
<span class="line"><span style="color:#FFCB6B;">./promtool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">check</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">rules</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">second_rules.yml</span></span></code></pre></div><h2 id="系统负载工具" tabindex="-1">系统负载工具 <a class="header-anchor" href="#系统负载工具" aria-label="Permalink to &quot;系统负载工具&quot;">​</a></h2><h3 id="stress" tabindex="-1">stress <a class="header-anchor" href="#stress" aria-label="Permalink to &quot;stress&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">sudo</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">yum</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">install</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">epel-release</span></span>
<span class="line"><span style="color:#FFCB6B;">sudo</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">yum</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">update</span></span>
<span class="line"><span style="color:#FFCB6B;">sudo</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">yum</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">install</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stress</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 模拟CPU负载，这会模拟4个CPU核心的负载，在60秒后停止。</span></span>
<span class="line"><span style="color:#FFCB6B;">stress</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--cpu</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">4</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--timeout</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">s</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 模拟内存负载，这会模拟使用512MB内存的虚拟机，持续60秒</span></span>
<span class="line"><span style="color:#FFCB6B;">stress</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--vm</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--vm-bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">512</span><span style="color:#C3E88D;">M</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--timeout</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">s</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 模拟IO负载，这会模拟4个IO操作，在60秒后停止。</span></span>
<span class="line"><span style="color:#FFCB6B;">stress</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--io</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">4</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--timeout</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">s</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 模拟磁盘负载，这会模拟写入1GB数据的磁盘操作，在60秒后停止</span></span>
<span class="line"><span style="color:#FFCB6B;">stress</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--hdd</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--hdd-bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#C3E88D;">G</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--timeout</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">s</span></span></code></pre></div><h3 id="lookbusy" tabindex="-1">lookbusy <a class="header-anchor" href="#lookbusy" aria-label="Permalink to &quot;lookbusy&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 下载编译安装lookbusy</span></span>
<span class="line"><span style="color:#FFCB6B;">wget</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://www.devin.com/lookbusy/download/lookbusy-1.4.tar.gz</span></span>
<span class="line"><span style="color:#FFCB6B;">tar</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-xzf</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">lookbusy-1.4.tar.gz</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">lookbusy-1.4</span></span>
<span class="line"><span style="color:#FFCB6B;">./configure</span></span>
<span class="line"><span style="color:#FFCB6B;">make</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&amp;</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">make</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">install</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 所有的cpu使用率都是30%</span></span>
<span class="line"><span style="color:#FFCB6B;">lookbusy</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-c</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">30</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 占用所有 CPU 核心各 70%</span></span>
<span class="line"><span style="color:#FFCB6B;">lookbusy</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-c</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">70</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 占用两个 CPU 核心各 70%</span></span>
<span class="line"><span style="color:#FFCB6B;">lookbusy</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-c</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">70</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-n</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">2</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 占用所有 CPU 核心在 60%-70% 上下浮动</span></span>
<span class="line"><span style="color:#FFCB6B;">lookbusy</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-c</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">-70</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-r</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">curve</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># cpu以60分钟为周期，30分钟是峰值，使用率在60%-70%上下浮动</span></span>
<span class="line"><span style="color:#FFCB6B;">lookbusy</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-c</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">-70</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--cpu-mode</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">curve</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--cpu-curve-period</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">60</span><span style="color:#C3E88D;">m</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--cpu-curve-peak</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">30</span><span style="color:#C3E88D;">m</span></span></code></pre></div><h1 id="prometheus" tabindex="-1">Prometheus <a class="header-anchor" href="#prometheus" aria-label="Permalink to &quot;Prometheus&quot;">​</a></h1><h2 id="概述" tabindex="-1">概述 <a class="header-anchor" href="#概述" aria-label="Permalink to &quot;概述&quot;">​</a></h2><blockquote><p>Prometheus是一个开源系统监控和警报工具包，受启发于Google的Brogmon监控系统（相似的Kubernetes是从Google的Brog系统演变而来），从2012年开始由前Google工程师在Soundcloud以开源软件的形式进行研发，并且于2015年早期对外发布早期版本。2016年5月继Kubernetes之后成为第二个正式加入CNCF基金会的项目，同年6月正式发布1.0版本。2017年底发布了基于全新存储层的2.0版本，能更好地与容器平台、云平台配合。</p></blockquote><h3 id="主要特点" tabindex="-1">主要特点 <a class="header-anchor" href="#主要特点" aria-label="Permalink to &quot;主要特点&quot;">​</a></h3><blockquote><p>支持多维数据模型由指标名称和键值对标识的时间序列数据</p><p><strong>内置时间序列库TSDB（Time Serices Database）</strong></p><p><strong>支持PromQL（Promethues Query Language），对数据的查询和分析、图形展示和监控告警</strong>。</p><p><strong>不依赖分布式存储</strong>；单个服务器节点是自治的</p><p><strong>支持HTTP 的拉取(pull)方式收集时间序列数据</strong></p><p>通过中间网关<strong>Pushgateway推送时间序列</strong></p><p><strong>通过服务发现或静态配置2种方式发现目标</strong></p><p><strong>支持多种可视化和仪表盘</strong>，如：grafana</p></blockquote><h3 id="核心组件" tabindex="-1">核心组件 <a class="header-anchor" href="#核心组件" aria-label="Permalink to &quot;核心组件&quot;">​</a></h3><blockquote><p>Prometheus Server， 主要用于抓取数据和存储时序数据，另外还提供查询和 Alert Rule 配置管理。</p><p>client libraries，用于检测应用程序代码的客户端库。</p><p>push gateway ，用于批量，短期的监控数据的汇总节点，主要用于业务数据汇报等。</p><p>exporters 收集监控样本数据，并以标准格式向 Prometheus 提供。例如：收集服务器系统数据的</p><p>node_exporter, 收集 MySQL 监控样本数据的是 MySQL exporter 等等。</p><p>alertmanager，用于告警通知管理的 。</p></blockquote><h3 id="基础架构" tabindex="-1">基础架构 <a class="header-anchor" href="#基础架构" aria-label="Permalink to &quot;基础架构&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262023960.png" alt="image-20230426202349845" style="zoom:80%;"><p>架构图看出 Prometheus 主要模块包含， Server, Exporters, Pushgateway, PromQL,Alertmanager, WebUI</p><p>它大致使用逻辑是这样：</p><blockquote><p>Prometheus server 定期从静态配置的 targets 或者服务发现的 targets 拉取数据（Targets是Prometheus采集Agent需要抓取的采集目标）</p></blockquote><blockquote><p>当新拉取的数据大于配置内存缓存区的时候，Prometheus 会将数据持久化到磁盘（如果使用 remote storage 将持久化到云端）。</p></blockquote><blockquote><p>Prometheus 可以配置 rules，然后定时查询数据，当条件触发的时候，会将 alerts 推送到配置的 Alertmanager</p><p>Alertmanager 收到警告的时候，可以根据配置（163，钉钉等），聚合，去重，降噪，最后发送警告。</p><p>可以使用 API， Prometheus Console 或者 Grafana 查询和聚合数据。</p></blockquote><h3 id="prometheus-vs-zabbix" tabindex="-1">Prometheus VS Zabbix <a class="header-anchor" href="#prometheus-vs-zabbix" aria-label="Permalink to &quot;Prometheus VS Zabbix&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262026608.png" alt="image-20230426202611516" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262026077.png" alt="image-20230426202628979" style="zoom:80%;"><blockquote><p>监控系统没有绝对的谁好谁不好，最重要的是适合自己的公司团队，能够合理利用最小的成本解决问题。</p><p>prometheus，zabbix 都只是工具，监控思想才是最重要的。实在不知道怎么选？参考如下：</p></blockquote><blockquote><p>物理机、硬件设备的监控推荐使用Zabbix</p><p>而Docker容器，Kubernetes监控推荐用Prometheus</p><p>云服务器厂商自带有监控系统，有的监控不全面，也可以搭配zabbix和Prometheus来一起使用</p></blockquote><h2 id="相关概念" tabindex="-1">相关概念 <a class="header-anchor" href="#相关概念" aria-label="Permalink to &quot;相关概念&quot;">​</a></h2><h3 id="时间序列" tabindex="-1">时间序列 <a class="header-anchor" href="#时间序列" aria-label="Permalink to &quot;时间序列&quot;">​</a></h3><blockquote><p>安装好prometheus后会暴露一个/metrics的HTTP服务（相当于安装了prometheus_exporter），通过配置（默认会加上/metrics），Prometheus就可以采集到这个/metrics里面所有监控样本数据。例如：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># HELP process_open_fds Number of open file descriptors.</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># TYPE process_open_fds gauge</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">25</span></span></code></pre></div><h3 id="样本" tabindex="-1">样本 <a class="header-anchor" href="#样本" aria-label="Permalink to &quot;样本&quot;">​</a></h3><blockquote><p>Prometheus 会将所有采集到的监控样本数据以时间序列的方式保存在<strong>内存数据库</strong>中，并且定时保存到硬盘上。时间序列是按照时间戳和值的序列顺序存放的，我们称之为向量(vector)，每条时间序列通过指标名称(metrics name)和一组标签集(label)命名。如下所示，可以将时间序列理解为一个以时间为 X 轴的数字矩阵：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">^</span></span>
<span class="line"><span style="color:#FFCB6B;">│</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">process_open_fds</span></span>
<span class="line"><span style="color:#FFCB6B;">│</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span></span>
<span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{cpu=</span><span style="color:#FFCB6B;">&quot;cpu0&quot;</span><span style="color:#FFCB6B;">,mode</span><span style="color:#A6ACCD;">=</span><span style="color:#FFCB6B;">&quot;system&quot;</span><span style="color:#FFCB6B;">}</span></span>
<span class="line"><span style="color:#FFCB6B;">│</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_load1{}</span></span>
<span class="line"><span style="color:#FFCB6B;">│</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.</span></span>
<span class="line"><span style="color:#FFCB6B;">v</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">------------------ 时间 ----------------</span><span style="color:#89DDFF;">&gt;</span></span></code></pre></div><p>在时间序列中的每一个点称为一个样本（sample），样本由以下三部分组成：</p><blockquote><p>指标(metric)：指标名和描述当前样本特征的标签集合</p><p>时间戳(timestamp)：一个精确到毫秒的时间戳</p><p>样本值(value)： 一个 float64 的浮点型数据表示当前样本的值</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">27</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">--------------------- metric ------------------------------&gt;&lt;-timestamp -&gt;&lt;-value-</span><span style="color:#89DDFF;">&gt;</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@1434417560938</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">39</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@1434417561287</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">33</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@1434417560938</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">35</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@1434417561287</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">37</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@1434417560938</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">36</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@1434417561287</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">25</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">--metric_name-&gt;&lt;----------------lable----------------------</span><span style="color:#89DDFF;">&gt;</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">--metric_name-&gt;&lt;--name--&gt;&lt;----value----</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">name&gt;&lt;--value---</span><span style="color:#89DDFF;">&gt;</span></span></code></pre></div><h3 id="指标-metric" tabindex="-1">指标(Metric) <a class="header-anchor" href="#指标-metric" aria-label="Permalink to &quot;指标(Metric)&quot;">​</a></h3><p>在形式上，所有的指标(Metric)都通过如下格式标示：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">metric name</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">{</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label name</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">=</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label value</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">, ...}</span></span></code></pre></div><blockquote><p>指标的名称(metric name)可以反映被监控样本的含义（比如， process_open_fds - 表示当前系统打开的文件描述符）。指标名称只能由ASCII字符、数字、下划线以及冒号组成并必须符合正则表达式 [a-zA-Z_:] [a-zA-Z0-9_:]* 。</p></blockquote><blockquote><p>标签(label)反映了当前样本的特征维度，通过这些维度Prometheus可以对样本数据进行过滤，聚合等。标签的名称只能由ASCII字符、数字以及下划线组成并满足正则表达式 [a-zA-Z_][a-zA-Z0-9_]* 。</p></blockquote><blockquote><p>其中以 __ 作为前缀的标签，是系统保留的关键字，只能在系统内部使用。标签的值则可以包含任何Unicode编码的字符。在Prometheus的底层实现中指标名称实际上是以 <strong>name</strong>=&lt;metric name&gt; 的形式保存在数据库中的，因此以下两种方式均表示的同一条time-series：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">__name__</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">process_open_fds</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">localhost:9090</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span></span></code></pre></div><h3 id="指标类型" tabindex="-1">指标类型 <a class="header-anchor" href="#指标类型" aria-label="Permalink to &quot;指标类型&quot;">​</a></h3><blockquote><p>Prometheus 底层存储上其实并没有对指标做类型的区分，都是以时间序列的形式存储，但是为了方便用户的使用和理解不同监控指标之间的差异，Prometheus 定义了 counter（计数器） 、gauge（仪表盘） 、histogram（直方图） 以及 summary （摘要）这四种 Metrics 类型。</p></blockquote><blockquote><p><strong>Gauge/Counter</strong> <strong>是数值指标，代表数据的变化情况，Histogram/Summary是统计类型的指标，表示数据的分布情况</strong>。在Exporter返回的样本数据中，其注释中也包含了该样本的类型。例如：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># HELP process_resident_memory_bytes Resident memory size in bytes.</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># TYPE process_resident_memory_bytes gauge</span></span>
<span class="line"><span style="color:#FFCB6B;">process_resident_memory_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1.3586432</span><span style="color:#C3E88D;">e+08</span></span></code></pre></div><h4 id="counter-只增不减的计数器" tabindex="-1">Counter：只增不减的计数器 <a class="header-anchor" href="#counter-只增不减的计数器" aria-label="Permalink to &quot;Counter：只增不减的计数器&quot;">​</a></h4><blockquote><p>Counter类型的指标其工作方式和计数器一样，只增不减（除非系统发生重置）。常见的监控指标，如node_cpu，http_requests_total都是Counter类型的监控指标。 一般在定义Counter类型指标的名称时推荐使用_total作为后缀。</p></blockquote><blockquote><p>通过 Counter 指标可以统计 HTTP 请求数量，请求错误数，接口调用次数等单调递增的数据，同时可结合increase 和 rate 等函数统计变化速率</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 通过PromQL内置的聚合rate()函数获取HTTP请求量的评价增长率</span></span>
<span class="line"><span style="color:#FFCB6B;">rate(prometheus_http_requests_total[5m]</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查询当前系统中，访问量前10的HTTP地址</span></span>
<span class="line"><span style="color:#FFCB6B;">topk(10,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus_http_requests_total</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262147824.png" alt="image-20230426214732686" style="zoom:80%;"><h4 id="gauge-可增可减的仪表盘" tabindex="-1">Gauge：可增可减的仪表盘 <a class="header-anchor" href="#gauge-可增可减的仪表盘" aria-label="Permalink to &quot;Gauge：可增可减的仪表盘&quot;">​</a></h4><blockquote><p>与Counter不同，Gauge类型的指标侧重于反应系统的当前状态。因此这类指标的样本数据可增可减。常见指标如：node_memory_MemFree_bytes（主机当前空闲的物理内存大小）、node_memory_MemAvailable_bytes （可用内存大小）都是Gauge类型的监控指标。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 通过Gauge指标，通过PromQL可以直接查看系统的当前空闲物理内存大小</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemFree_bytes</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262148547.png" alt="image-20230426214826417" style="zoom:80%;"><blockquote><p>对于Gauge类型的监控指标，通过PromQL内置函数delta()可以获取样本在一段时间返回内的变化情况。例如，计算CPU温度在两个小时内的差异：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">delta(cpu_temp_celsius</span><span style="color:#A6ACCD;">{host=</span><span style="color:#FFCB6B;">&quot;zeus&quot;</span><span style="color:#FFCB6B;">}[2h]</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><blockquote><p>还可以使用deriv()计算样本的线性回归模型，甚至是直接使用predict_linear()对数据的变化趋势进行预测。例如，预测系统磁盘空间在4个小时之后的剩余情况：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">predict_linear(node_filesystem_avail_bytes</span><span style="color:#A6ACCD;">{}</span><span style="color:#FFCB6B;">[1h],</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">4</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">3600</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><h4 id="分析数据分布情况" tabindex="-1">分析数据分布情况 <a class="header-anchor" href="#分析数据分布情况" aria-label="Permalink to &quot;分析数据分布情况&quot;">​</a></h4><blockquote><p>除了Counter和Gauge类型的监控指标以外，Prometheus还定义了Histogram和Summary的指标类型。Histogram和Summary主用用于统计和分析样本的分布情况。</p></blockquote><blockquote><p>在大多数情况下人们都倾向于使用某些量化指标的平均值，例如CPU的平均使用率、页面的平均响应时间。这种方式的问题很明显，以系统API调用的平均响应时间为例：如果大多数API请求都维持在100ms的响应时间范围内，而个别请求的响应时间需要5s，那么就会导致某些WEB页面的响应时间落到中位数的情况，这种现象被称为长尾问题</p></blockquote><blockquote><p>为了区分是平均的慢还是长尾的慢，最简单的方式就是按照请求延迟的范围进行分组。例如，统计延迟在0~10ms之间的请求数有多少而10~20ms之间的请求数又有多少。通过这种方式可以快速分析系统慢的原因。Histogram和Summary都是为了能够解决这样问题的存在，通过Histogram和Summary类型的监控指标，我们可以快速了解监控样本的分布情况。</p></blockquote><blockquote><p>例如，指标prometheus_tsdb_wal_fsync_duration_seconds的指标类型为Summary。 它记录了Prometheus Server中wal_fsync处理的处理时间，通过访问Prometheus Server的/metrics地址，可以获取到以下监控样本数据：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># HELP prometheus_tsdb_wal_fsync_duration_seconds Duration of WAL fsync.</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># TYPE prometheus_tsdb_wal_fsync_duration_seconds summary</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_wal_fsync_duration_seconds</span><span style="color:#A6ACCD;">{quantile=</span><span style="color:#FFCB6B;">&quot;0.5&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0.012352463</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_wal_fsync_duration_seconds</span><span style="color:#A6ACCD;">{quantile=</span><span style="color:#FFCB6B;">&quot;0.9&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0.014458005</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_wal_fsync_duration_seconds</span><span style="color:#A6ACCD;">{quantile=</span><span style="color:#FFCB6B;">&quot;0.99&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0.017316173</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_wal_fsync_duration_seconds_sum</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">2.888716127000002</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_wal_fsync_duration_seconds_count</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">216</span></span></code></pre></div><p>从上面的样本中可以得知当前Prometheus Server进行wal_fsync操作的总次数为216次，耗时2.888716127000002s。其中中位数（quantile=0.5）的耗时为0.012352463，9分位数（quantile=0.9）的耗时为0.014458005s。</p><p>在Prometheus Server自身返回的样本数据中，我们还能找到类型为Histogram的监控指标</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># HELP prometheus_tsdb_compaction_chunk_range_seconds Final time range of chunks on</span></span>
<span class="line"><span style="color:#FFCB6B;">their</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">first</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">compaction</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># TYPE prometheus_tsdb_compaction_chunk_range_seconds histogram</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;100&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;400&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;1600&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;6400&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;25600&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">61</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;102400&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1095</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;409600&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1208</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;1.6384e+06&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">3184</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;6.5536e+06&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">217665</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;2.62144e+07&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">217695</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_bucket</span><span style="color:#A6ACCD;">{le=</span><span style="color:#FFCB6B;">&quot;+Inf&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">217695</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_sum</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">3.91254640158</span><span style="color:#C3E88D;">e+11</span></span>
<span class="line"><span style="color:#FFCB6B;">prometheus_tsdb_compaction_chunk_range_seconds_count</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">217695</span></span></code></pre></div><blockquote><p>与Summary类型的指标相似之处在于Histogram类型的样本同样会反应当前指标的记录的总数(以_count作为后缀)以及其值的总量（以_sum作为后缀）。不同在于Histogram指标直接反应了在不同区间内样本的个数，区间通过标签len进行定义。</p></blockquote><blockquote><p>同时对于Histogram的指标，我们还可以通过histogram_quantile()函数计算出其值的分位数。不同在于Histogram通过histogram_quantile函数是在服务器端计算的分位数。 而Sumamry的分位数则是直接在客户端计算完成。因此对于分位数的计算而言，Summary在通过PromQL进行查询时有更好的性能表现，而Histogram则会消耗更多的资源。反之对于客户端而言Histogram消耗的资源更少。在选择这两种方式时用户应该按照自己的实际场景进行选择。</p></blockquote><p>需要特别注意的是，假设采样数据 metric 叫做 x(指标名) , 如果 x 是 histogram 或 summary 类型必需满足以下条件：</p><blockquote><p>采样数据的总和应表示为 x_sum 。</p><p>采样数据的总量应表示为 x_count 。</p><p>summary 类型的采样数据的 quantile 应表示为 x{quantile=&quot;y&quot;} 。</p><p>histogram 类型的采样分区统计数据将表示为 x_bucket{le=&quot;y&quot;} 。</p><p>histogram 类型的采样必须包含 x_bucket{le=&quot;+Inf&quot;} , 它的值等于 x_count 的值。summary 和 historam 中 quantile 和 le 必需按从小到大顺序排列。</p></blockquote><h3 id="job-任务-和instances-实例" tabindex="-1">job（任务）和instances（实例） <a class="header-anchor" href="#job-任务-和instances-实例" aria-label="Permalink to &quot;job（任务）和instances（实例）&quot;">​</a></h3><blockquote><p>在Prometheus中，任何被采集的目标，即每一个暴露监控样本数据的HTTP服务都称为一个实例（Instance）,例如在当前主机上运行的node exporter可以被称为一个实例(Instance)。而具有相同采集目的的实例集合称为任务Job</p></blockquote><h4 id="job-任务" tabindex="-1">job（任务） <a class="header-anchor" href="#job-任务" aria-label="Permalink to &quot;job（任务）&quot;">​</a></h4><p>例如，以下2个复制实例的node作业：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> job: node</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">2</span><span style="color:#C3E88D;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1.2</span><span style="color:#C3E88D;">.3.4:9100</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">4</span><span style="color:#C3E88D;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">5.6</span><span style="color:#C3E88D;">.7.8:9100</span></span></code></pre></div><h4 id="instances-实例" tabindex="-1">instances（实例） <a class="header-anchor" href="#instances-实例" aria-label="Permalink to &quot;instances（实例）&quot;">​</a></h4><blockquote><p>通过在prometheus.yml配置文件中，添加如下配置。我们让Prometheus可以从node exporter暴露的服务中获取监控指标数据。</p></blockquote><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">scrape_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">prometheus:9090</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">node-exporter:9100</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.11.62:9100</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><p>当我们需要采集不同的监控指标(例如：主机、MySQL、Nginx)时，我们只需要运行相应的监控采集程序，在Prometheus Server配置这些Exporter实例的访问地址。</p><h4 id="实例状态" tabindex="-1">实例状态 <a class="header-anchor" href="#实例状态" aria-label="Permalink to &quot;实例状态&quot;">​</a></h4><p>除了通过使用“up”表达式查询当前所有Instance的状态以外，还可以通过Prometheus UI中的Targets页面查看当前所有的监控采集任务，以及各个任务下所有实例的状态:</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262156085.png" alt="image-20230426215654952" style="zoom:80%;"><h2 id="exporter" tabindex="-1">Exporter <a class="header-anchor" href="#exporter" aria-label="Permalink to &quot;Exporter&quot;">​</a></h2><p><a href="https://prometheus.io/docs/instrumenting/exporters/#databases" target="_blank" rel="noreferrer">https://prometheus.io/docs/instrumenting/exporters/#databases</a></p><p><a href="https://github.com/prometheus-community" target="_blank" rel="noreferrer">https://github.com/prometheus-community</a></p><h3 id="概述-1" tabindex="-1">概述 <a class="header-anchor" href="#概述-1" aria-label="Permalink to &quot;概述&quot;">​</a></h3><blockquote><p>所有可以向Prometheus提供监控样本数据的程序都可以被称为⼀个Exporter。⽽Exporter的⼀个实例称为target，如下所示，Prometheus通过轮询的⽅式定期从这些target中获取样本数据:</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261717029.png" alt="image-20230426171704919" style="zoom:80%;"><blockquote><p>注：安装好Exporter后会暴露⼀个 <a href="http://ip" target="_blank" rel="noreferrer">http://ip</a>:端⼝/metrics 的HTTP服务</p><p>通过Prometheus添加配置 -targets: [&#39;node_exporter:9100&#39;] （默认会加上/metrics）</p><p>Prometheus就可以采集到这个<a href="http://ip" target="_blank" rel="noreferrer">http://ip</a>:端⼝/metrics ⾥⾯所有监控样本数据</p></blockquote><h3 id="来源" tabindex="-1">来源 <a class="header-anchor" href="#来源" aria-label="Permalink to &quot;来源&quot;">​</a></h3><p>从Exporter的来源上来讲，主要分为两类：社区提供的、⽤户⾃定义的</p><p>社区提供的( <a href="https://prometheus.io/docs/instrumenting/exporters/" target="_blank" rel="noreferrer">https://prometheus.io/docs/instrumenting/exporters/</a> )</p><h4 id="社区提供" tabindex="-1">社区提供 <a class="header-anchor" href="#社区提供" aria-label="Permalink to &quot;社区提供&quot;">​</a></h4><blockquote><p>Prometheus社区提供了丰富的Exporter实现，涵盖了从基础设施，中间件以及⽹络等各个⽅⾯的监控功能。这些Exporter可以实现⼤部分通⽤的监控需求。下表列举⼀些社区中常⽤的Exporter：</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2022.8.30/202209092016928.png" alt="image-20220909201647860" style="zoom:80%;"><h4 id="用户自定义" tabindex="-1">用户自定义 <a class="header-anchor" href="#用户自定义" aria-label="Permalink to &quot;用户自定义&quot;">​</a></h4><blockquote><p>除了直接使⽤社区提供的Exporter程序以外，⽤户还可以基于Prometheus提供的Client Library创建⾃⼰的Exporter程序，⽬前Promthues社区官⽅提供了对以下编程语⾔的⽀持：Go、Java/Scala、Python、Ruby。同时还有第三⽅实现的如：Bash、C++、Common Lisp、Erlang,、Haskeel、Lua、Node.js、PHP、Rust等。</p></blockquote><h3 id="exporter-类型" tabindex="-1">Exporter 类型 <a class="header-anchor" href="#exporter-类型" aria-label="Permalink to &quot;Exporter 类型&quot;">​</a></h3><p>通常来说可以将Exporter分为两类：</p><h4 id="直接采集型" tabindex="-1">直接采集型 <a class="header-anchor" href="#直接采集型" aria-label="Permalink to &quot;直接采集型&quot;">​</a></h4><blockquote><p>这类Exporter直接内置了相应的应⽤程序，⽤于向Prometheus直接提供Target数据⽀持。这样设计的好处是，可以更好地监控各⾃系统的内部运⾏状态，同时也适合更多⾃定义监控指标的项⽬实施。例如cAdvisor、Kubernetes等，它们均内置了⽤于向Prometheus提供监控数据的端点。</p></blockquote><h4 id="间接采集型" tabindex="-1">间接采集型 <a class="header-anchor" href="#间接采集型" aria-label="Permalink to &quot;间接采集型&quot;">​</a></h4><blockquote><p>原始监控⽬标并不直接⽀持Prometheus，需要我们使⽤Prometheus提供的Client Library编写该监控⽬标的监控采集程序，⽤户可以将该程序独⽴运⾏，去获取指定的各类监控数据值。例如，由于Linux操作系统⾃身并不能直接⽀持Prometheus，⽤户⽆法从操作系统层⾯上直接提供对Prometheus的⽀持，因此单独安装Node exporter，还有数据库或⽹站HTTP应⽤类等Exporter。</p></blockquote><h3 id="exporter-规范" tabindex="-1">Exporter 规范 <a class="header-anchor" href="#exporter-规范" aria-label="Permalink to &quot;Exporter 规范&quot;">​</a></h3><blockquote><p>所有的Exporter程序都需要按照Prometheus的规范，返回监控的样本数据。以Node Exporter为例，当访问：<a href="http://192.168.11.61:9100/metrics%E5%9C%B0%E5%9D%80%E6%97%B6%E4%BC%9A%E8%BF%94%E5%9B%9E%E4%BB%A5%E4%B8%8B%E5%86%85%E5%AE%B9%EF%BC%9A" target="_blank" rel="noreferrer">http://192.168.11.61:9100/metrics地址时会返回以下内容：</a></p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261722470.png" alt="image-20230426172235347" style="zoom:67%;"><ul><li>以#开始的⾏通常都是注释内容。这些样本数据集合说明如下：</li><li>以#HELP开始的⾏，表示metric的帮助与说明注释，可以包含当前监控指标名称和对应的说明信息。</li><li>以#TYPE开始的⾏，表示定义metric类型，可以包含当前监控指标名称和类型，类型有Counter、Gauge、Histogram、Summary和Untyped。</li><li>⾮#开头的⾏，就是监控样本数据</li></ul><h3 id="监控样本数据规范" tabindex="-1">监控样本数据规范 <a class="header-anchor" href="#监控样本数据规范" aria-label="Permalink to &quot;监控样本数据规范&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">metric_name</span><span style="color:#A6ACCD;"> [</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">&quot;{&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">label_name</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">`</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#89DDFF;">`</span><span style="color:#C3E88D;"> </span><span style="color:#FFCB6B;">label_value</span><span style="color:#C3E88D;"> </span><span style="color:#89DDFF;">`</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#89DDFF;">`</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">&quot;,&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">label_name</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">`</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#89DDFF;">`</span></span>
<span class="line"><span style="color:#FFCB6B;">label_value</span><span style="color:#C3E88D;"> </span><span style="color:#89DDFF;">`</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#89DDFF;">`</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">]</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">] value [ timestamp ]</span></span></code></pre></div><blockquote><p>其中metric_name和label_name必须遵循PromQL的格式规范要求。value是⼀个float格式的数据，timestamp的类型为int64（从1970-01-01 00:00:00以来的毫秒数），timestamp为可选默认为当前时间。具有相同metric_name的样本必须按照⼀个组的形式排列，并且每⼀⾏必须是唯⼀的指标名称和标签键值对组合。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">go_memstats_mcache_sys_bytes</span><span style="color:#A6ACCD;">{} </span><span style="color:#F78C6C;">15600</span></span>
<span class="line"><span style="color:#FFCB6B;">go_memstats_mcache_sys_bytes</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">timestamp</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">VALUE</span></span></code></pre></div><h1 id="promql" tabindex="-1">PromQL <a class="header-anchor" href="#promql" aria-label="Permalink to &quot;PromQL&quot;">​</a></h1><blockquote><p>Prometheus 通过指标名称（metrics name）以及对应的一组标签（labelset）唯一定义一条时间序列。指标名称反映了监控样本的基本标识，而 label 则在这个基本特征上为采集到的数据提供了多种特征维度。用户可以基于这些特征维度过滤，聚合，统计从而产生新的计算后的一条时间序列。PromQL 是 Prometheus <strong>内置的数据查询语言</strong>，其提供对时间序列数据丰富的查询，聚合以及逻辑运算能力的支持。并且被广泛应用在 Prometheus的日常应用当中，包括对数据查询、可视化、告警处理当中。可以这么说，PromQL 是Prometheus 所有应用场景的基础，理解和掌握 PromQL 是 Prometheus 入门的第一课。<a href="https://prometheus.io/docs/prometheus/latest/querying/api/" target="_blank" rel="noreferrer">HTTP API中使用PromQL</a></p></blockquote><p><a href="https://prometheus.io/docs/prometheus/latest/querying/basics/" target="_blank" rel="noreferrer">官网地址</a></p><h2 id="数据类型" tabindex="-1">数据类型 <a class="header-anchor" href="#数据类型" aria-label="Permalink to &quot;数据类型&quot;">​</a></h2><p>在Prometheus的表达式语言中，PromQL数据类型归类为以下四种：</p><blockquote><p><strong>瞬时向量（instant vector）</strong>，是指同一时刻的一组时间序列，每个时间序列包含一个样本，所有样本共享相同的时间戳，即每个时序只有一个点。</p><p><strong>区间向量（range vector）</strong>，是指在任何一个时间范围内的一组时间序列，包含每个时间序列随时间变化的一系列数据点，这时每个时序有多个点。</p><p><strong>标量（scalar）</strong>，即纯量数据，一个简单的数字浮点值，只有一个数字，没有时序。</p><p><strong>字符串（string）</strong>，一个目前未被使用的简单字符串值。</p></blockquote><h3 id="瞬时向量-instant-vector" tabindex="-1">瞬时向量（Instant vector） <a class="header-anchor" href="#瞬时向量-instant-vector" aria-label="Permalink to &quot;瞬时向量（Instant vector）&quot;">​</a></h3><p>我们在Prometheus的查询页面输入指标 node_memory_Active_bytes 进行查询。返回如下的结果：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270918730.png" alt="image-20230427091832593" style="zoom:80%;"><h3 id="区间向量-range-vector" tabindex="-1">区间向量（Range vector） <a class="header-anchor" href="#区间向量-range-vector" aria-label="Permalink to &quot;区间向量（Range vector）&quot;">​</a></h3><p>我们在Prometheus的查询页面输入一个查询语句 node_memory_Active_bytes[1m] 进行查询。返回如下的结果：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270919770.png" alt="image-20230427091907621" style="zoom:80%;"><blockquote><p>以上的查询语句是查询 最近一分钟数据，因为采集的周期是 15s 所以[1m]每一个时序会有4个数据。区间向量（Range vector）类型数据是指时序有一个时间区间的数据，这种类型的数据成为区间向量（Range vector）。</p></blockquote><h3 id="标量-scalar" tabindex="-1">标量（Scalar） <a class="header-anchor" href="#标量-scalar" aria-label="Permalink to &quot;标量（Scalar）&quot;">​</a></h3><p>标量只有一个数字，没有时序。</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">1024</span></span></code></pre></div><blockquote><p>需要注意的是，当使用表达式count(http_requests_total)，返回的数据类型，依然是瞬时向量。用户可以通过内置函数scalar()将单个瞬时向量转换为标量。</p></blockquote><h3 id="字符串-string" tabindex="-1">字符串（String） <a class="header-anchor" href="#字符串-string" aria-label="Permalink to &quot;字符串（String）&quot;">​</a></h3><p>直接使用字符串，作为PromQL表达式，则会直接返回字符串。(目前未被使用)</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">&quot;this is a string&quot;</span></span>
<span class="line"><span style="color:#FFCB6B;">&#39;these are unescaped: \n \\ \t&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">`</span><span style="color:#FFCB6B;">these</span><span style="color:#C3E88D;"> are not unescaped: </span><span style="color:#A6ACCD;">\n</span><span style="color:#C3E88D;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;"> &quot; \t`</span></span></code></pre></div><h2 id="时间序列过滤器" tabindex="-1">时间序列过滤器 <a class="header-anchor" href="#时间序列过滤器" aria-label="Permalink to &quot;时间序列过滤器&quot;">​</a></h2><h3 id="瞬时向量过滤器" tabindex="-1">瞬时向量过滤器 <a class="header-anchor" href="#瞬时向量过滤器" aria-label="Permalink to &quot;瞬时向量过滤器&quot;">​</a></h3><blockquote><p>瞬时向量过滤器允许在指定的时间戳内选择一组时间序列和每个时间序列的单个样本值。在最简单的形式中，近指定指标（metric）名称。这将生成包含此指标名称的所有时间序列的元素的瞬时向量。</p></blockquote><p>例如：选择指标名称为 node_cpu_seconds_total 的所有时间序列：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">node_cpu_seconds_total</span></span></code></pre></div><p>可以通过向花括号 {} 里附加一组标签来进一步过滤时间序列。</p><p>例：选择指标名称为 node_cpu_seconds_total ， instance 标签值为 Prometheus服务器 ， mode 标签值为 idle 的时间序列</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">node_cpu_seconds_total{ instance=&quot;Prometheus服务器&quot;, mode=&quot;idle&quot;}</span></span></code></pre></div><p>PromQL 还支持用户根据时间序列的标签匹配模式来对时间序列进行过滤，目前主要支持两种匹配模式：完全匹配和正则匹配。总共有以下几种标签匹配运算符：</p><blockquote><p>= : 选择与提供的字符串完全相同的标签。</p><p>!= : 选择与提供的字符串不相同的标签。</p><p>=~ : 选择正则表达式与提供的字符串（或子字符串）相匹配的标签。</p><p>!~ : 选择正则表达式与提供的字符串（或子字符串）不匹配的标签。</p></blockquote><p>例如：选择指标名称为 node_cpu_seconds_total ，mode为 idle 、 user 或 system ， cpu!=0 的时间序列：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">node_cpu_seconds_total{mode=~&quot;idle|user|system&quot;,cpu!=&quot;0&quot;}</span></span>
<span class="line"><span style="color:#A6ACCD;">node_cpu_seconds_total{cpu=&quot;0&quot;}</span></span></code></pre></div><blockquote><p>没有指定标签的标签过滤器会选择该指标名称的所有时间序列。Prometheus 中的所有正则表达式都使用 RE2语法。所有的 PromQL 表达式必须至少包含一个指标名称，或者一个不会匹配到空字符串的标签过滤器。</p></blockquote><blockquote><p>除了使用 &lt;metric name&gt;{label=value} 的形式以外，我们还可以使用内置的 <strong>name</strong> 标签来指定监控指标名称。例如：表达式 node_cpu_seconds_total 等效于 {<strong>name</strong>=&quot;node_cpu_seconds_total&quot;}</p></blockquote><h3 id="区间向量过滤器" tabindex="-1">区间向量过滤器 <a class="header-anchor" href="#区间向量过滤器" aria-label="Permalink to &quot;区间向量过滤器&quot;">​</a></h3><p>区间向量与瞬时向量的工作方式类似，唯一的差异在于在区间向量表达式中我们需要定义时间选择的范围，时间范围通过时间范围选择器 [] 进行定义，以指定应为每个返回的区间向量样本值中提取多长的时间范围。时间范围通过数字来表示，单位可以使用以下其中之一的时间单位：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270924085.png" alt="image-20230427092423949" style="zoom:80%;"><p>例如：选择在过去 5 分钟内指标名称为 node_cpu_seconds_total ， job 标签值为 prometheus 的所有时间序列：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{ </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}[</span><span style="color:#F78C6C;">5</span><span style="color:#C3E88D;">m]</span></span></code></pre></div><h3 id="时间位移操作" tabindex="-1">时间位移操作 <a class="header-anchor" href="#时间位移操作" aria-label="Permalink to &quot;时间位移操作&quot;">​</a></h3><p>在瞬时向量表达式或者区间向量表达式中，都是以当前时间为基准：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{} </span><span style="color:#676E95;font-style:italic;"># 瞬时向量表达式，选择当前最新的数据</span></span>
<span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{}</span><span style="color:#FFCB6B;">[5m]</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 区间向量表达式，选择以当前时间为基准，5分钟内的数据</span></span></code></pre></div><p>而如果我们想查询，5 分钟前的瞬时样本数据，或昨天一天的区间内的样本数据呢? 这个时候我们就可以使用位移操作，位移操作的关键字为 offset 。例如，以下表达式返回相对于当前查询时间过去 5 分钟的 node_cpu_seconds_total 值：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{ </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">offset</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">5</span><span style="color:#C3E88D;">m</span></span>
<span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{ </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node1:9100</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">offset</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">5</span><span style="color:#C3E88D;">m</span></span></code></pre></div><p><strong>注意：</strong> offset 关键字需要紧跟在选择器 {} 后面。以下表达式是正确的：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">sum(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{ </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">offset</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">5</span><span style="color:#C3E88D;">m</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 下面的表达式是不合法的：</span></span>
<span class="line"><span style="color:#FFCB6B;">sum(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{ </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;">) offset 5m</span></span></code></pre></div><p>该操作同样适用于区间向量。以下表达式返回指标 node_cpu_seconds_total 一周前的 5 分钟之内的 cpu空闲率：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 最后的1w表示一周</span></span>
<span class="line"><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{ </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}[</span><span style="color:#F78C6C;">5</span><span style="color:#C3E88D;">m]</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">offset</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#C3E88D;">w</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><p>sum是聚合运算符，rate是内置函数，后面会讲</p><h2 id="操作符" tabindex="-1">操作符 <a class="header-anchor" href="#操作符" aria-label="Permalink to &quot;操作符&quot;">​</a></h2><p>Prometheus 的查询语言支持基本的逻辑运算和算术运算。对于两个瞬时向量, 匹配行为可以被改变。</p><h3 id="二元运算符" tabindex="-1">二元运算符 <a class="header-anchor" href="#二元运算符" aria-label="Permalink to &quot;二元运算符&quot;">​</a></h3><blockquote><p>在 Prometheus 系统中支持下面的二元算术运算符：+ 加法、- 减法、* 乘法、/ 除法、% 模、^ 幂等。二元运算操作符支持 scalar/scalar(标量/标量) 、 vector/scalar(瞬时向量/标量) 、和vector/vector(瞬时向量/瞬时向量) 之间的操作。在两个标量之间进行数学运算，得到的结果也是标量。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">63+3</span><span style="color:#89DDFF;">)*</span><span style="color:#A6ACCD;">3 ---</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> scalar 198</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301047628.png" alt="image-20230430104701523" style="zoom:80%;"><p>瞬时向量和标量之间进行算数运算时，算术运算符会一次作用与瞬时向量中的每一个样本值，得到一组新的时间序列</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 例如：我们通过监控指标node_memory_MemTotal_bytes（主机内存总大小）单位为byte,如果换成mb时</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemTotal_bytes/(1024*1024</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 结果</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">3901.07421875</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">3901.04296875</span></span></code></pre></div><p>瞬时向量与瞬时向量之间进行数学运算时，过程会相对复杂一点，运算符会依次找到与左边向量元素匹配（标签完全一致）的右边向量元素进行运算，如果没找到匹配元素，则直接丢弃。同时新的时间序列将不会包含指标名称。</p><p>例如，如果我们 node_memory_MemAvailable_bytes （可用内存） 和 node_memory_MemTotal_bytes（总内存） 获取内存可用率%，可以使用如下表达式：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemTotal_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">100</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 结果</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">73.19077778055924</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">69.0442359665434</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 案例 幂运算</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">^</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">3</span></span></code></pre></div><h3 id="关系运算符" tabindex="-1">关系运算符 <a class="header-anchor" href="#关系运算符" aria-label="Permalink to &quot;关系运算符&quot;">​</a></h3><p>目前，Prometheus 支持以下关系运算符：</p><p>== (相等)</p><p>!= (不相等)</p><p>&gt; (大于)</p><p>&lt; (小于)</p><p>&gt;= (大于等于)</p><p>&lt;= (小于等于)</p><blockquote><p>关系运算符被应用于 scalar/scalar（标量/标量） 、 vector/scalar（瞬时向量/标量） ，和vector/vector（瞬时向量/向量） 。默认情况下关系运算符只会根据时间序列中样本的值，对时间序列进行过滤。我们可以通过在运算符后面使用 bool 修饰符来改变关系运算的默认行为。使用 bool 修改符后，关系运算不会对时间序列进行过滤，而是直接依次瞬时向量中的各个样本数据与标量的比较结果 0 或者 1 。</p></blockquote><blockquote><p>在两个标量之间进行关系运算，必须提供 bool 修饰符，得到的结果也是标量，即 0 （ false ）或 1（ true ）。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">2</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">bool</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 结果为 1</span></span></code></pre></div><blockquote><p>瞬时向量和标量之间的关系运算，这个运算符会应用到某个当前时刻的每个时序数据上，如果一个时序数据的样本值与这个标量比较的结果是 false ，则这个时序数据被丢弃掉，如果是 true , 则这个时序数据被保留在结果中。如果提供了 bool 修饰符，那么比较结果是 0 的时序数据被丢弃掉，而比较结果是 1 的时序数据被保留。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_load1</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 结果为 true 或 false</span></span>
<span class="line"><span style="color:#FFCB6B;">node_load1</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">bool</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 结果为 1 或 0</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">==</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">!=</span><span style="color:#F78C6C;">1</span></span></code></pre></div><blockquote><p>瞬时向量与瞬时向量直接进行关系运算时，同样遵循默认的匹配模式：依次找到与左边向量元素匹配（标签完全一致）的右边向量元素进行相应的操作，如果没找到匹配元素，或者计算结果为 false，则直接丢弃。如果匹配上了，则将左边向量的度量指标和标签的样本数据写入瞬时向量。如果提供了 bool 修饰符，那么比较结果是 0 的时序数据被丢弃掉，而比较结果是 1 的时序数据（只保留左边向量）被保留。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">bool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemTotal_bytes</span></span>
<span class="line"><span style="color:#FFCB6B;">或</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">bool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemTotal_bytes</span></span></code></pre></div><h3 id="逻辑运算符" tabindex="-1">逻辑运算符 <a class="header-anchor" href="#逻辑运算符" aria-label="Permalink to &quot;逻辑运算符&quot;">​</a></h3><blockquote><p>使用瞬时向量表达式能够获取到一个包含多个时间序列的集合，我们称为瞬时向量。 通过集合运算，可以在两个瞬时向量与瞬时向量之间进行相应的集合操作。目前，Prometheus 支持以下集合运算符：and、or 、unless (排除)</p></blockquote><blockquote><p><strong>vector1 and vector2</strong> 会产生一个由vector1的元素组成的新的向量。该向量包含vector1中完全匹配vector2中的元素</p></blockquote><blockquote><p><strong>vector1 or vector2</strong> 会产生一个新的向量，该向量包含vector1的所有原始元素（标签集+值）的向量，以及vector2中没有与vector1匹配标签集的所有元素。</p></blockquote><blockquote><p><strong>vector1 unless vector2</strong> 会产生一个由vector1的元素组成的向量，而这些元素在vector2中没有与标签集完全匹配的元素，两个向量中的所有匹配元素都被删除。</p></blockquote><p>and表达式</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=~</span><span style="color:#FFCB6B;">&quot;idle|user|system&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">and</span></span>
<span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=~</span><span style="color:#FFCB6B;">&quot;user|system|iowait&quot;</span><span style="color:#FFCB6B;">}</span></span></code></pre></div><p>时钟不同步触发器</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">min_over_time(node_timex_sync_status[1m]</span><span style="color:#A6ACCD;">) == 0 and node_timex_maxerror_seconds </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">=16</span></span></code></pre></div><p>or表达式</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=~</span><span style="color:#FFCB6B;">&quot;idle|user|system&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">or</span></span>
<span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=~</span><span style="color:#FFCB6B;">&quot;user|system|iowait&quot;</span><span style="color:#FFCB6B;">}</span></span></code></pre></div><p>状态码非200-399触发器</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">probe_http_status_code</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">199</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">OR</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">probe_http_status_code</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">400</span></span></code></pre></div><p>unless表达式</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=~</span><span style="color:#FFCB6B;">&quot;idle|user|system&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">unless</span></span>
<span class="line"><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=~</span><span style="color:#FFCB6B;">&quot;user|system|iowait&quot;</span><span style="color:#FFCB6B;">}</span></span></code></pre></div><h3 id="优先级" tabindex="-1">优先级 <a class="header-anchor" href="#优先级" aria-label="Permalink to &quot;优先级&quot;">​</a></h3><p>在 Prometheus 系统中，二元运算符优先级从高到低的顺序为：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270912029.png" alt="image-20230427091234898" style="zoom:80%;"><blockquote><p>具有相同优先级的运算符是满足结合律的（左结合）。例如， 2 * 3 % 2 等价于 (2 * 3) % 2 。运算符^ 例外， ^ 满足的是右结合，例如， 2 ^ 3 ^ 2 等价于 2 ^ (3 ^ 2) 。</p></blockquote><h2 id="向量匹配" tabindex="-1">向量匹配 <a class="header-anchor" href="#向量匹配" aria-label="Permalink to &quot;向量匹配&quot;">​</a></h2><p>在标量和瞬时向量之间使用运算符可以满足很多需求，但是在两个瞬时向量之间使用运算符时，哪些样本应该适用于哪些其他样本？这种瞬时向量的匹配称为向量匹配。Prometheus提供了两种基本的向量匹配模式：one-to-one向量匹配和many-to-one（one-to-many）向量匹配。接下来将介绍在 PromQL 中有两种典型的匹配模式：一对一（one-to-one）,多对一（many-to-one）或一对多（one-to-many）。</p><h3 id="一对一-one-to-one" tabindex="-1">一对一（one-to-one） <a class="header-anchor" href="#一对一-one-to-one" aria-label="Permalink to &quot;一对一（one-to-one）&quot;">​</a></h3><p>一对一匹配模式会从操作符两边表达式获取的瞬时向量依次比较并找到唯一匹配(标签完全一致)的样本值。默认情况下，使用表达式：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">vector1 &lt;operator&gt; vector2</span></span></code></pre></div><p>例如当存在样本：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">process_open_fds{instance=&quot;Prometheus服务器&quot;, job=&quot;node-exporter&quot;} 9</span></span>
<span class="line"><span style="color:#A6ACCD;">process_max_fds{instance=&quot;Prometheus服务器&quot;, job=&quot;node-exporter&quot;} 1048576</span></span></code></pre></div><p>使用 PromQL 案例：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">process_open_fds{instance=&quot;Prometheus服务器&quot;, job=&quot;node-exporter&quot;} /</span></span>
<span class="line"><span style="color:#A6ACCD;">process_max_fds{instance=&quot;Prometheus服务器&quot;, job=&quot;node-exporter&quot;}</span></span></code></pre></div><p>因此结果如下：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">{instance=&quot;Prometheus服务器&quot;, job=&quot;node-exporter&quot;} 0.00000858306884765625</span></span></code></pre></div><p>在操作符两边表达式标签不一致的情况下，可以使用 on(label list) 或者 ignoring(label list） 来修改便签的匹配行为。使用 ignoreing 可以在匹配时忽略某些便签。而 on 则用于将匹配行为限定在某些便签之内。</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">&lt;vector expr&gt; &lt;bin-op&gt; ignoring(&lt;label list&gt;) &lt;vector expr&gt;</span></span>
<span class="line"><span style="color:#A6ACCD;">&lt;vector expr&gt; &lt;bin-op&gt; on(&lt;label list&gt;) &lt;vector expr&gt;</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270906942.png" alt="image-20230427090646778" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270907358.png" alt="image-20230427090701209" style="zoom:80%;"><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">sum by(instance, job)(rate(node_cpu_seconds_total{mode=&quot;idle&quot;}[5m]))</span></span>
<span class="line"><span style="color:#A6ACCD;">sum by(instance)(rate(node_cpu_seconds_total[5m]))</span></span></code></pre></div><p>使用on表达式：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">sum by(instance, job)(rate(node_cpu_seconds_total{mode=&quot;idle&quot;}[5m])) / on(instance)</span></span>
<span class="line"><span style="color:#A6ACCD;">sum by(instance)(rate(node_cpu_seconds_total[5m]))</span></span></code></pre></div><p>结果</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">{instance=&quot;Prometheus服务器&quot;} 1.0199580591510582</span></span>
<span class="line"><span style="color:#A6ACCD;">{instance=&quot;test服务器&quot;} 1.0159356093344827</span></span></code></pre></div><h3 id="多对一和一对多" tabindex="-1">多对一和一对多 <a class="header-anchor" href="#多对一和一对多" aria-label="Permalink to &quot;多对一和一对多&quot;">​</a></h3><blockquote><p>多对一和一对多的匹配模式，可以理解为向量元素中的一个样本数据匹配到了多个样本数据标签。在使用该匹配模式时，需要使用group_left或group_right修饰符明确指定哪一个向量具有更高的基数，也就是说左或者右决定了哪边的向量具有较高的子集。此表达格式为：</p></blockquote><div class="language-sql"><button title="Copy Code" class="copy"></button><span class="lang">sql</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">bin</span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;">op</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> ignoring(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) group_left(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">bin</span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;">op</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> ignoring(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) group_right(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">bin</span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;">op</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">on</span><span style="color:#A6ACCD;">(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) group_left(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">bin</span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;">op</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">on</span><span style="color:#A6ACCD;">(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) group_right(</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">vector expr</span><span style="color:#89DDFF;">&gt;</span></span></code></pre></div><p>多对一和一对多两种模式一定是出现在操作符两侧表达式返回的向量标签不一致的情况。因此需要使用 ignoring 和 on 修饰符来排除或者限定匹配的标签列表。使用group_left指定左侧操作数组中可以有多个匹配样本，例如</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">sum without(cpu)(rate(node_cpu_seconds_total[5m])) / ignoring(mode) group_left sum</span></span>
<span class="line"><span style="color:#A6ACCD;">without(mode, cpu)(rate(node_cpu_seconds_total[5m]))</span></span></code></pre></div><p>结果：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.9802134676075311</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">iowait</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}0.000035459735470373514</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">irq</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">nice</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.000017729867735189905</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">softirq</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.0003368674869685515</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">steal</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">system</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.008049359951775699</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">Prometheus服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">user</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.01134711535051912</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.9848852662354926</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">iowait</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.00001771950031008937</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">irq</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">nice</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">softirq</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.0007796580136441839</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">steal</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">system</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.00425268007442165</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">test服务器</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">user</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">0.010064676176131668</span></span></code></pre></div><p>左右两边的瞬时向量标签不一致：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270911847.png" alt="image-20230427091116682" style="zoom:80%;"><h2 id="内置函数" tabindex="-1">内置函数 <a class="header-anchor" href="#内置函数" aria-label="Permalink to &quot;内置函数&quot;">​</a></h2><h3 id="数学函数" tabindex="-1">数学函数 <a class="header-anchor" href="#数学函数" aria-label="Permalink to &quot;数学函数&quot;">​</a></h3><blockquote><p>数学函数对瞬时向量执⾏标准的数学运算，如计算绝对值或取对数。瞬时向量中的每个样本都是独⽴处理的，并且在返回值中删除指标名称。</p></blockquote><p>abs：绝对值</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># abs该函数输⼊,瞬时向量，返回其每个值的绝对值。例如，对于表达式process_open_fds，调试后返回信息为：49</span></span>
<span class="line"><span style="color:#FFCB6B;">process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,job</span><span style="color:#A6ACCD;">=</span><span style="color:#FFCB6B;">&quot;prometheus&quot;</span><span style="color:#FFCB6B;">}</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#  使⽤函数表达式求绝对值后返回信息为，72</span></span>
<span class="line"><span style="color:#FFCB6B;">abs(process_open_fds</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;localhost:9090&quot;</span><span style="color:#FFCB6B;">,job</span><span style="color:#A6ACCD;">=</span><span style="color:#FFCB6B;">&quot;prometheus&quot;</span><span style="color:#FFCB6B;">}-119</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><p>sqrt：该函数返回瞬时向量中值的平⽅根，例如表达式 sqrt(vector(25)) ，返回结果为5</p><p>round：该函数将瞬时向量中的值四舍五⼊到最近的整数。例如表达式 round(vector(7.5)) ，返回结果为：8</p><h3 id="时间函数" tabindex="-1">时间函数 <a class="header-anchor" href="#时间函数" aria-label="Permalink to &quot;时间函数&quot;">​</a></h3><blockquote><p>Prometheus使⽤的是协调世界时（UTC），没有时区的概念。为了使⽤户在使⽤中不⽤⾃⼰实现与⽇期相关的逻辑，Prometheus提供了⼀些时间函数。该函数是最基本的时间函数，它将查询的计算时间以秒为单位返回。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 例如，查看进程运⾏了多⻓时间，可以使⽤表达式：</span></span>
<span class="line"><span style="color:#82AAFF;">time</span><span style="color:#89DDFF;">()</span><span style="color:#A6ACCD;">-process_start_time_seconds</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 结果</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">localhost:9090</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">3137.62700009346</span></span>
<span class="line"><span style="color:#89DDFF;">{</span><span style="color:#A6ACCD;">instance</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node_exporter:9100</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> job</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">node-exporter</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#FFCB6B;">3138.367000102997</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 触发器案例:证书还有30天到期</span></span>
<span class="line"><span style="color:#FFCB6B;">probe_ssl_earliest_cert_expiry</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">time</span><span style="color:#89DDFF;">()</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">86400</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">30</span></span></code></pre></div><h3 id="计算counter指标增⻓率函数" tabindex="-1">计算Counter指标增⻓率函数 <a class="header-anchor" href="#计算counter指标增⻓率函数" aria-label="Permalink to &quot;计算Counter指标增⻓率函数&quot;">​</a></h3><blockquote><p>我们知道Counter类型的监控指标其特点是只增不减，在没有发⽣重置（如服务器重启，应⽤重启）的情况下其样本值应该是不断增⼤的。为了能够更直观++的表示样本数据的变化剧烈情况，需要计算样本的增⻓速率。</p></blockquote><p>如下图所示，样本增⻓率反映出了样本变化的剧烈程度：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262206458.png" alt="image-20230426220610311" style="zoom:80%;"><p>通过增⻓率表示样本的变化情况</p><p>increase(v range-vector)函数是PromQL中提供的众多内置函数之⼀。其中参数v是⼀个区间向量，</p><p>increase函数获取区间向量中的第⼀个后最后⼀个样本并返回其增⻓量。因此，可以通过以下表达式</p><p>Counter类型指标的增⻓率：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">increase(node_load1[2m]</span><span style="color:#A6ACCD;">) / 120</span></span></code></pre></div><blockquote><p>这⾥通过node_cpu[2m]获取时间序列最近两分钟的所有样本，increase计算出最近两分钟的增⻓量，最后除以时间120秒得到node_cpu样本在最近两分钟的平均增⻓率。这个值也近似于主机节点最近两分钟内的平均CPU使⽤率</p></blockquote><blockquote><p>除了使⽤increase函数以外，PromQL中还直接内置了rate(v range-vector)函数，rate函数可以直接计算区间向量v在时间窗⼝内平均增⻓速率。因此，通过以下表达式可以得到与increase函数相同的结果：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">rate(node_load1[2m]</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><blockquote><p>需要注意的是使⽤rate或者increase函数去计算样本的平均增⻓速率，容易陷⼊“⻓尾问题”当中，其⽆法反应在时间窗⼝内样本数据的突发变化。 例如，对于主机⽽⾔在2分钟的时间窗⼝内，可能在某⼀个由于访问量或者其它问题导致CPU占⽤100%的情况，但是通过计算在时间窗⼝内的平均增⻓率却⽆法反应出该问题。</p></blockquote><blockquote><p>为了解决该问题，PromQL提供了另外⼀个灵敏度更⾼的函数irate(v range-vector)。irate同样⽤于计算区间向量的计算率，但是其反应出的是瞬时增⻓率。irate函数是通过区间向量中最后两个样本数据来计算区间向量的增⻓速率。这种⽅式可以避免在时间窗⼝范围内的“⻓尾问题”，并且体现出更好的灵敏度，通过irate函数绘制的图标能够更好的反应样本数据的瞬时变化状态。</p></blockquote><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">irate(node_load1[2m])</span></span></code></pre></div><blockquote><p>irate函数相⽐于rate函数提供了更⾼的灵敏度，不过当需要分析⻓期趋势或者在告警规则中，irate的这种灵敏度反⽽容易造成⼲扰。因此在⻓期趋势分析或者告警中更推荐使⽤rate函数。</p></blockquote><h3 id="预测gauge指标变化趋势函数" tabindex="-1">预测Gauge指标变化趋势函数 <a class="header-anchor" href="#预测gauge指标变化趋势函数" aria-label="Permalink to &quot;预测Gauge指标变化趋势函数&quot;">​</a></h3><blockquote><p>在⼀般情况下，系统管理员为了确保业务的持续可⽤运⾏，会针对服务器的资源设置相应的告警阈值。例如，当磁盘空间只剩512MB时向相关⼈员发送告警通知。 这种基于阈值的告警模式对于当资源⽤量是平滑增⻓的情况下是能够有效的⼯作的。 但是如果资源不是平滑变化的呢？ ⽐如有些某些业务增⻓，存储空间的增⻓速率提升了⾼⼏倍。这时，如果基于原有阈值去触发告警，当系统管理员接收到告警以后可能还没来得及去处理问题，系统就已经不可⽤了。</p></blockquote><blockquote><p>因此阈值通常来说不是固定的，需要定期进⾏调整才能保证该告警阈值能够发挥去作⽤。 那么还有没有更好的⽅法吗？PromQL中内置的predict_linear(v range-vector, t scalar) 函数可以帮助系统管理员更好的处理此类情况，</p></blockquote><blockquote><p>predict_linear函数可以预测时间序列v在t秒后的值。它基于简单线性回归的⽅式，对时间窗⼝内的样本数据进⾏统计，从⽽可以对时间序列的变化趋势做出预测。例如，基于1⼩时的样本数据，来预测主机可⽤磁盘空间的是否在24个⼩时候被占满，可以使⽤如下表达式：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">predict_linear(node_filesystem_avail_bytes</span><span style="color:#A6ACCD;">{fstype!~</span><span style="color:#FFCB6B;">&quot;tmpfs&quot;</span><span style="color:#FFCB6B;">}[1h],</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">24</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">3600</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;"> 0</span></span></code></pre></div><p>node_exporter触发器解释</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">node_filesystem_avail_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">100</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> / node_filesystem_size_bytes </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;"> 10 and ON </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">device,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mountpoint</span><span style="color:#89DDFF;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">predict_linear(node_filesystem_avail_bytes</span><span style="color:#A6ACCD;">{fstype!~</span><span style="color:#FFCB6B;">&quot;tmpfs&quot;</span><span style="color:#FFCB6B;">}[1h],</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">24</span><span style="color:#A6ACCD;"> </span><span style="color:#A6ACCD;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">3600</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;"> 0 and ON </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">device,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mountpoint</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> node_filesystem_readonly == 0</span></span></code></pre></div><p>磁盘可⽤率&lt;10% 并且 基于1⼩时的样本数据，来预测主机可⽤磁盘空间的是否在24个⼩时候被占满 并且 ⽂件系统不等于只读</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">node_filesystem_readonly ：只读⽂件系统</span></span>
<span class="line"><span style="color:#A6ACCD;">node_filesystem_avail_bytes : 磁盘剩余空间⼤⼩（字节）</span></span>
<span class="line"><span style="color:#A6ACCD;">node_filesystem_size_bytes : 磁盘总⼤⼩（字节）</span></span></code></pre></div><h3 id="标签操作函数" tabindex="-1">标签操作函数 <a class="header-anchor" href="#标签操作函数" aria-label="Permalink to &quot;标签操作函数&quot;">​</a></h3><p>⼀般来说来说，使⽤PromQL查询到时间序列后，可视化⼯具会根据时间序列的标签来渲染图表。例如通过up指标可以获取到当前所有运⾏的Exporter实例以及其状态：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">up</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301214269.png" alt="image-20230430121423076" style="zoom:80%;"><p>这是可视化⼯具渲染图标时可能根据，instance和job的值进⾏渲染，为了能够让客户端的图标更具有可读性，可以通过<strong>label_replace标签为时间序列添加额外的标签</strong>。label_replace的具体参数如下：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">label_replace(v</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instant-vector,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">dst_label</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">replacement</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">src_label</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">regex</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><p>该函数会依次对v中的每⼀条时间序列进⾏处理，通过regex匹配src_label的值，并将匹配部分relacement 写⼊到dst_label标签中。如下所示：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 新增host标签名</span></span>
<span class="line"><span style="color:#FFCB6B;">label_replace(up,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">host</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;font-style:italic;">$1</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">instance</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">(.*):.*</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><p>函数处理后，时间序列将包含⼀个host标签，host标签的值为Exporter实例的IP地址：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;192.168.11.61&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_icmp</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{host=</span><span style="color:#FFCB6B;">&quot;192.168.11.61&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">192.168.11.61:22</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_tcp</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{host=</span><span style="color:#FFCB6B;">&quot;192.168.11.61&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">192.168.11.61:9090</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_tcp</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;192.168.11.62&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_icmp</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{host=</span><span style="color:#FFCB6B;">&quot;192.168.11.62&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">192.168.11.62:8081</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">springboot-demo</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{host=</span><span style="color:#FFCB6B;">&quot;192.168.11.62&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">192.168.11.62:9222</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">domain</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span>
<span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{host=</span><span style="color:#FFCB6B;">&quot;192.168.11.62&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">192.168.11.62:9256</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">process</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span></span></code></pre></div><p>除了label_replace以外，Prometheus还提供了label_join函数，该函数可以将时间序列中v多个标签src_label的值，通过separator作为连接符写⼊到⼀个新的标签dst_label中:</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">label_join(v</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instant-vector,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">dst_label</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">separator</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">src_label_1</span></span>
<span class="line"><span style="color:#FFCB6B;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">src_label_2</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">string,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">...</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><p>表达式:</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">up</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;192.168.11.61&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_icmp</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span></span></code></pre></div><p>使⽤label_join表达式：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">label_join(up</span><span style="color:#A6ACCD;">{instance=</span><span style="color:#FFCB6B;">&quot;192.168.11.61&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_icmp</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">},</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">ip</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">,</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">job</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">label_join(up,</span><span style="color:#FFCB6B;">&quot;ip&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#FFCB6B;">&quot;,&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#FFCB6B;">&quot;job&quot;</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><p>label_replace和label_join函数提供了对时间序列标签的⾃定义能⼒，从⽽能够更好的于客户端或者可视化⼯具配合。</p><h3 id="其它内置函数" tabindex="-1">其它内置函数 <a class="header-anchor" href="#其它内置函数" aria-label="Permalink to &quot;其它内置函数&quot;">​</a></h3><blockquote><p>除了上⽂介绍的这些内置函数以外，PromQL还提供了⼤量的其它内置函数。这⾥就不⼀⼀细讲，感兴趣的同学可以通过阅读Prometheus的官⽅⽂档，了解这些函数的使⽤⽅式。<a href="https://prometheus.io/docs/prometheus/latest/querying/functions/" target="_blank" rel="noreferrer">官方文档</a></p></blockquote><h2 id="聚合操作" tabindex="-1">聚合操作 <a class="header-anchor" href="#聚合操作" aria-label="Permalink to &quot;聚合操作&quot;">​</a></h2><p>Prometheus 还提供了下列内置的聚合操作符，这些操作符作用域瞬时向量。可以将瞬时表达式返回的样本数据进行聚合，形成一个具有较少样本值的新的时间序列。</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">sum</span><span style="color:#A6ACCD;"> (求和)</span></span>
<span class="line"><span style="color:#FFCB6B;">min</span><span style="color:#A6ACCD;"> (最小值)</span></span>
<span class="line"><span style="color:#FFCB6B;">max</span><span style="color:#A6ACCD;"> (最大值)</span></span>
<span class="line"><span style="color:#FFCB6B;">avg</span><span style="color:#A6ACCD;"> (平均值)</span></span>
<span class="line"><span style="color:#FFCB6B;">stddev</span><span style="color:#A6ACCD;"> (标准差)</span></span>
<span class="line"><span style="color:#FFCB6B;">stdvar</span><span style="color:#A6ACCD;"> (标准差异)</span></span>
<span class="line"><span style="color:#FFCB6B;">count</span><span style="color:#A6ACCD;"> (计数)</span></span>
<span class="line"><span style="color:#FFCB6B;">count_values</span><span style="color:#A6ACCD;"> (对 </span><span style="color:#C3E88D;">value</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">进行计数</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">bottomk</span><span style="color:#A6ACCD;"> (样本值最小的 </span><span style="color:#C3E88D;">k</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">个元素</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">topk</span><span style="color:#A6ACCD;"> (样本值最大的k个元素)</span></span>
<span class="line"><span style="color:#FFCB6B;">quantile</span><span style="color:#A6ACCD;"> (分布统计)</span></span></code></pre></div><p>这些操作符被用于聚合所有标签维度，或者通过 without 或者 by 子语句来保留不同的维度。</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">aggr-op</span><span style="color:#89DDFF;">&gt;([</span><span style="color:#C3E88D;">parameter,</span><span style="color:#89DDFF;">]</span><span style="color:#C3E88D;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#C3E88D;">vector expression</span><span style="color:#89DDFF;">&gt;)</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;">without</span><span style="color:#89DDFF;">|</span><span style="color:#A6ACCD;">by (</span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">label list</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">)</span><span style="color:#89DDFF;">]</span></span></code></pre></div><blockquote><p>其中只有 count_values , quantile , topk , bottomk 支持参数(parameter)。without（排除标签名称）by（保留标签名称）： 类似sql的：goup by</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 查询系统所有 http 请求的总量</span></span>
<span class="line"><span style="color:#FFCB6B;">sum(prometheus_http_requests_total</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 按照 mode 计算主机 CPU 的平均使用时间</span></span>
<span class="line"><span style="color:#FFCB6B;">avg(node_cpu_seconds_total</span><span style="color:#A6ACCD;">) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">mode</span><span style="color:#89DDFF;">)</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 按照主机查询各个主机的 CPU 使用率</span></span>
<span class="line"><span style="color:#FFCB6B;">sum(sum(irate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode!=</span><span style="color:#FFCB6B;">&#39;idle&#39;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#A6ACCD;">)) / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">irate(node_cpu_seconds_total</span><span style="color:#A6ACCD;"> [5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">)) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span></span></code></pre></div><h3 id="sum结果求和" tabindex="-1">sum结果求和 <a class="header-anchor" href="#sum结果求和" aria-label="Permalink to &quot;sum结果求和&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 下面两种相同</span></span>
<span class="line"><span style="color:#FFCB6B;">sum(node_cpu_seconds_total</span><span style="color:#A6ACCD;">) without </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">cpu,job,mode</span><span style="color:#89DDFF;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">sum(node_cpu_seconds_total</span><span style="color:#A6ACCD;">) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span></span></code></pre></div><p>计算prometheus http请求总量，表达式：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">sum(prometheus_http_requests_total</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><h3 id="max-最大值" tabindex="-1"><strong>max</strong> 最大值 <a class="header-anchor" href="#max-最大值" aria-label="Permalink to &quot;**max** 最大值&quot;">​</a></h3><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">max(node_cpu_seconds_total) by (mode)</span></span></code></pre></div><h3 id="avg-平均值" tabindex="-1">avg 平均值 <a class="header-anchor" href="#avg-平均值" aria-label="Permalink to &quot;avg 平均值&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">avg(node_cpu_seconds_total</span><span style="color:#A6ACCD;">) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">mode</span><span style="color:#89DDFF;">)</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># cpu负载&gt; 80%触发器</span></span>
<span class="line"><span style="color:#FFCB6B;">100</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> (avg </span><span style="color:#C3E88D;">by</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> (rate(node_cpu_seconds_total{mode</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">idle</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}[</span><span style="color:#F78C6C;">2</span><span style="color:#C3E88D;">m]</span><span style="color:#A6ACCD;">)) </span><span style="color:#FFCB6B;">*</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">100</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> 80</span></span></code></pre></div><h3 id="count-统计数量" tabindex="-1">count（统计数量) <a class="header-anchor" href="#count-统计数量" aria-label="Permalink to &quot;count（统计数量)&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">count(prometheus_http_requests_total</span><span style="color:#A6ACCD;">)</span></span></code></pre></div><h3 id="bottomk-统计最小的几个值" tabindex="-1">bottomk （统计最小的几个值） <a class="header-anchor" href="#bottomk-统计最小的几个值" aria-label="Permalink to &quot;bottomk （统计最小的几个值）&quot;">​</a></h3><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">bottomk(3, sum(node_cpu_seconds_total) by (mode))</span></span></code></pre></div><h3 id="topk-统计最大的几个值" tabindex="-1">topk （统计最大的几个值) <a class="header-anchor" href="#topk-统计最大的几个值" aria-label="Permalink to &quot;topk （统计最大的几个值)&quot;">​</a></h3><p>类似sql：ORDER BY vaule DESC limit 3</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">topk(3,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">node_cpu_seconds_total</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">by</span><span style="color:#A6ACCD;"> (mode))</span></span></code></pre></div><h3 id="基于时间聚合" tabindex="-1">基于时间聚合 <a class="header-anchor" href="#基于时间聚合" aria-label="Permalink to &quot;基于时间聚合&quot;">​</a></h3><blockquote><p>前面我们已经学习了如何使用 sum() 、 avg() 和相关的聚合运算符从标签维度进行聚合，这些运算符在一个时间内对多个序列进行聚合，但是有时候我们可能想在每个序列中按时间进行聚合，例如，使尖锐的曲线更平滑，或深入了解一个序列在一段时间内的最大值。为了基于时间来计算这些聚合，PromQL 提供了一些与标签聚合运算符类似的函数，但是在这些函数名前面附加了 _over_time() ：</p></blockquote><blockquote><p>avg_over_time(range-vector) ：区间向量内每个指标的平均值。</p><p>min_over_time(range-vector) ：区间向量内每个指标的最小值。</p><p>max_over_time(range-vector) ：区间向量内每个指标的最大值。</p><p>sum_over_time(range-vector) ：区间向量内每个指标的求和。</p><p>count_over_time(range-vector) ：区间向量内每个指标的样本数据个数。</p><p>quantile_over_time(scalar, range-vector) ：区间向量内每个指标的样本数据值分位数。</p><p>stddev_over_time(range-vector) ：区间向量内每个指标的总体标准差。</p><p>stdvar_over_time(range-vector) ：区间向量内每个指标的总体标准方差。</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># max_over_time</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># MySQL运行的线程过多触发器，1分钟最大的值</span></span>
<span class="line"><span style="color:#FFCB6B;">max_over_time(mysql_global_status_threads_running[1m]</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> 20</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># avg_over_time</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># HTTP请求慢告警触发器，1分钟平均的值</span></span>
<span class="line"><span style="color:#FFCB6B;">avg_over_time(probe_duration_seconds[1m]</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> 1</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># min_over_time</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 时钟不同步告警触发器，1分钟最小的值</span></span>
<span class="line"><span style="color:#FFCB6B;">min_over_time(node_timex_sync_status[1m]</span><span style="color:#A6ACCD;">) == 0</span></span></code></pre></div><h1 id="grafana" tabindex="-1">Grafana <a class="header-anchor" href="#grafana" aria-label="Permalink to &quot;Grafana&quot;">​</a></h1><blockquote><p>Grafana 技术应用于海量指标数据的分析和展示场景，它是一个跨平台的开源度量分析和可视化工具，作为云原生可观测性的统一展示解决方案，其主要特性包括灵活且易用性强的客户端图表，丰富的仪表盘插件，具备实时查询海量指标数据并以热图、折线图、图表等多种方式进行展示的能力。这其中包括支持指定指标、图表类型、时间范围以及数据聚合方式的自定义配置以满足各种具体监控场景的需要。包括 Graphite、InfluxDB、OpenTSDB、Prometheus 和 Elasticsearch 等不同时序的数据源接入，并在同一面板展示不同数据源，将来自不同数据源，但同一业务的数据集中展示。通过 Grafana，任何人都可以创建和共享动态仪表盘，以促进协作和透明度，高效协同数据分析、故障排除。</p></blockquote><blockquote><p>Grafana官方文档：<a href="https://grafana.com/docs/grafana/latest/getting-started/getting-started-prometheus/" target="_blank" rel="noreferrer">https://grafana.com/docs/grafana/latest/getting-started/getting-started-prometheus/</a></p><p>node-exporter的使用：<a href="https://prometheus.io/docs/guides/node-exporter/" target="_blank" rel="noreferrer">https://prometheus.io/docs/guides/node-exporter/</a></p><p>访问：<a href="http://192.168.22.130:3000" target="_blank" rel="noreferrer">http://192.168.22.130:3000</a></p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2022.8.30/202209091259810.png" alt="image-20220909125958707" style="zoom:80%;"><h2 id="面板" tabindex="-1">面板 <a class="header-anchor" href="#面板" aria-label="Permalink to &quot;面板&quot;">​</a></h2><h3 id="面板概述" tabindex="-1">面板概述 <a class="header-anchor" href="#面板概述" aria-label="Permalink to &quot;面板概述&quot;">​</a></h3><blockquote><p>⾯板（Panel）是 Grafana 中基本可视化构建块，每个⾯板都有⼀个特定于⾯板中选择数据源的查询编辑器，每个⾯板都有各种各样的样式和格式选项，⾯板可以在仪表板上拖放和重新排列，它们也可以调整⼤⼩，所以要在 Grafana 上创建可视化的图表，⾯板是我们必须要掌握的知识点。</p></blockquote><blockquote><p>Panel 是 Grafana 中最基本的可视化单元，每⼀种类型的⾯板都提供了相应的查询编辑器(Query Editor)，让⽤户可以从不同的数据源（如 Prometheus）中查询出相应的监控数据，并且以可视化的⽅式展现，Grafana 中所有的⾯板均以插件的形式进⾏使⽤。Grafana 提供了各种可视化来⽀持不同的⽤例</p></blockquote><blockquote><p>⽬前内置⽀持的⾯板包括：Time series（时间序列）是默认的也是主要的图形可视化⾯板、State timeline（状态时间表）状态随时间变化 、Status history（状态历史记录）、Bar chart（条形图）、Histogram（直⽅图）、Heatmap（热⼒图）、Pie chart（饼状图）、Stat（统计数据）、Gauge、Bar gauge、Table（表格）、Logs（⽇志）、Node Graph（节点图）、Dashboard list（仪表板列表）、 Alert list（报警列表）、Text panel（⽂本⾯板，⽀持 markdown 和 html）、News Panel（新闻⾯板，可以显示 RSS 摘要）等</p></blockquote><blockquote><p>除此之外，我们还可以通过官⽹的⾯板插件⻚⾯<a href="https://grafana.com/grafana/plugins/?type=panel" target="_blank" rel="noreferrer">https://grafana.com/grafana/plugins/?type=panel</a> 获取安装其他⾯板进⾏使⽤。<a href="https://grafana.com/grafana/dashboards/" target="_blank" rel="noreferrer">https://grafana.com/grafana/dashboards/</a></p></blockquote><h3 id="常见面板⭐" tabindex="-1">常见面板⭐ <a class="header-anchor" href="#常见面板⭐" aria-label="Permalink to &quot;常见面板⭐&quot;">​</a></h3><h4 id="图表" tabindex="-1">图表 <a class="header-anchor" href="#图表" aria-label="Permalink to &quot;图表&quot;">​</a></h4><p>对于基于时间的折线图、⾯积图和条形图，我们建议使⽤默认的 Time series 时间序列进⾏可视化。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261600138.png" alt="image-20230426160027056" style="zoom:80%;"><h4 id="条形图" tabindex="-1">条形图 <a class="header-anchor" href="#条形图" aria-label="Permalink to &quot;条形图&quot;">​</a></h4><p>使⽤barchart⾯板进⾏可视化</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261600749.png" alt="image-20230426160048667" style="zoom:80%;"><h4 id="数据统计" tabindex="-1">数据统计 <a class="header-anchor" href="#数据统计" aria-label="Permalink to &quot;数据统计&quot;">​</a></h4><p>使⽤ Stat ⾯板进⾏可视化</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261601314.png" alt="image-20230426160120219" style="zoom:80%;"><h4 id="仪表盘" tabindex="-1">仪表盘 <a class="header-anchor" href="#仪表盘" aria-label="Permalink to &quot;仪表盘&quot;">​</a></h4><p>如下所示的标准径 Gauge ⾯板。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261611047.png" alt="image-20230426161130958" style="zoom:80%;"><h4 id="表格" tabindex="-1">表格 <a class="header-anchor" href="#表格" aria-label="Permalink to &quot;表格&quot;">​</a></h4><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261611084.png" alt="image-20230426161150994" style="zoom:80%;"><h4 id="饼状图" tabindex="-1">饼状图 <a class="header-anchor" href="#饼状图" aria-label="Permalink to &quot;饼状图&quot;">​</a></h4><p>Grafana 现在⽀持 Pie Chart 饼状图⾯板可视化。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261612396.png" alt="image-20230426161208304" style="zoom:80%;"><h4 id="热力图" tabindex="-1">热⼒图 <a class="header-anchor" href="#热力图" aria-label="Permalink to &quot;热⼒图&quot;">​</a></h4><p>要显示值分布，请使⽤ Heatmap 热⼒图⾯板可视化。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261612532.png" alt="image-20230426161235434" style="zoom:80%;"><h4 id="地图" tabindex="-1">地图 <a class="header-anchor" href="#地图" aria-label="Permalink to &quot;地图&quot;">​</a></h4><p>使⽤Goemap⾯板</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261612999.png" alt="image-20230426161253899" style="zoom:80%;"><h2 id="组成部分" tabindex="-1">组成部分 <a class="header-anchor" href="#组成部分" aria-label="Permalink to &quot;组成部分&quot;">​</a></h2><p>Grafana 主要包括以下几个部分</p><h3 id="数据源" tabindex="-1">数据源 <a class="header-anchor" href="#数据源" aria-label="Permalink to &quot;数据源&quot;">​</a></h3><p>对于 Grafana 而言，Prometheus 这类为其提供数据的对象均称为数据源（Data Source）。目前，Grafana 官方提供了对 Graphite、InfluxDB、OpenTSDB、Prometheus、Elasticsearch,、CloudWatch 的支持。对于 Grafana 管理员而言，只需要将这些对象以数据源的形式添加到 Grafana 中，Grafana 便可以轻松实现对这些数据的可视化工作。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2022.8.30/202209091300267.png" alt="image-20220909130029197" style="zoom:67%;"><h3 id="仪表盘-1" tabindex="-1">仪表盘 <a class="header-anchor" href="#仪表盘-1" aria-label="Permalink to &quot;仪表盘&quot;">​</a></h3><p>通过数据源定义好可视化的数据来源之后，可以通过数据看板来组织和管理数据可视化图表。在数据看板中最基本的可视化单元为一个 Panel（面板）。Panel 通过如趋势图，热力图的形式展示可视化数据。并且在数据看板中每个 Panel 是一个完全独立的部分。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2022.8.30/202209091303098.png" alt="image-20220909130307032" style="zoom:80%;"><p>通过 Panel 的 Query Editor（查询编辑器）我们可以为每一个 Panel 自己查询的数据源以及数据查询方式，以 Prometheus 作为数据源举例，那在 Query Editor 中，实际使用的是 PromQL，而 Panel 则会负责从特定的 Prometheus 中查询出相应数据并可视化。由于每个 Panel 完全独立，因此在一个数据看板中，往往可能会包含来自多个数据源的数据。Grafana 通过插件形式提供了多种 Panel 实现如：Graph Panel、Heatmap Panel、SingleStat Panel、Table Panel 等。用户还可通过插件安装更多类型 Panel 面板。</p><p><img src="https://mmbiz.qpic.cn/sz_mmbiz_png/qdzZBE73hWtgNhRxBcSgdHUbibZzaicZx9aHHhOKY4FtoVxbhYhs3PROo46qH9weO6F9oSQmPN6kIeBhAjzXG0kw/640?wx_fmt=png&amp;wxfrom=5&amp;wx_lazy=1&amp;wx_co=1" alt="图片"></p><h3 id="组织管理" tabindex="-1">组织管理 <a class="header-anchor" href="#组织管理" aria-label="Permalink to &quot;组织管理&quot;">​</a></h3><p>作为一个可视化工具，Grafana 除了提供灵活的可视化定制能力以外，还提供面向企业的组织级管理能力。在 Grafana 中，数据看板是属于一个组织。例如企业创建多个组织，用户可以属于一个或多个不同组织。并且在不同组织下，为用户赋予不同的权限，根据企业组织架构定义整个管理模型。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2022.8.30/202209091303657.png" alt="image-20220909130332595" style="zoom:80%;"><p>看到这里，大抵会觉得 Grafana 开源版本简直是最佳选择，但在平台搭建过程中，运维人依旧会遇到很多开源通病</p><ul><li><p>搭建前期，购买机器、配置网络、构建环境、安装部署、准备域名和 IP 地址等前期繁琐的准备工作；</p></li><li><p>使用过程中，使用了非常多云服务之后，数据接入困难，不知如何下手；</p></li><li><p>由于 SLA 无法保障，在关键时刻监控平台服务无法使用；</p></li><li><p>想要连接专有网络 VPC 内各自建数据源，并通过邮件、短信等渠道进行告警触达，需要自行详细配置。</p></li></ul><h3 id="工作原理" tabindex="-1">工作原理 <a class="header-anchor" href="#工作原理" aria-label="Permalink to &quot;工作原理&quot;">​</a></h3><p>上面说到，Grafana 是一个仪表盘，而仪表盘必然是用来显示数据的。</p><p>Grafana 本身并不负责数据层，它只提供了通用的接口，让底层的数据库可以把数据给它。而我们起的另一个服务，叫 Prometheus （中文名普罗米修斯数据库）则是负责存储和查询数据的。</p><p>也就是说，Grafana 每次要展现一个仪表盘的时候，会向 Prometheus 发送一个查询请求。</p><p>那么配置里的另一个服务 <code>Prometheus-exporter</code> 又是什么呢？</p><p>这个就是你真正监测的数据来源了，<code>Prometheus-exporter</code> 这个服务，会查询你的本地电脑的信息，比如内存还有多少、CPU 负载之类，然后将数据导出至普罗米修斯数据库。</p><p>在真实世界中，你的目的是监控你自己的服务，比如你的 Web 服务器，你的数据库之类。</p><p>那么你就需要在你自己的服务器中把数据发送给普罗米修斯数据库。当然，你完全可以把数据发送给 MySQL (Grafana 也支持)，但普罗米修斯几乎是标配的时序数据库，强烈建议你用。</p><p>用来说明它们之间的关系：</p><p><img src="https://mmbiz.qpic.cn/mmbiz_png/qFG6mghhA4bczBT86snY2mNdj6vHrUCXudPfEriaO4RzpHXaFxrOEG6DhfaB1iaVBamqzwibECmF18PjHq9BTcKFA/640?wx_fmt=png&amp;wxfrom=5&amp;wx_lazy=1&amp;wx_co=1" alt="图片"></p><p>这里，最左边的 Docker 服务会将服务的数据发送给中间的普罗米修斯（对应上文的 <code>Prometheus-exporter</code>），而最右边的 Grafana 会查询中间的普罗米修斯，来展示仪表盘。</p><h2 id="添加数据源" tabindex="-1">添加数据源 <a class="header-anchor" href="#添加数据源" aria-label="Permalink to &quot;添加数据源&quot;">​</a></h2><p>点击配置，点击 Data Sources：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201558457.png" alt="image-20230420155857337" style="zoom:80%;"><p>点击添加按钮：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201559463.png" alt="image-20230420155925378" style="zoom:80%;"><p>找到 Prometheus，点击 Select</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201559703.png" alt="image-20230420155951618" style="zoom:80%;"><p>配置 Prometheus Server 地址：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201601247.png" alt="image-20230420160100132" style="zoom:80%;"><p>点击下方的 Save&amp;Test：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201601401.png" alt="image-20230420160132315" style="zoom:80%;"><p>出现绿色的提示框，表示与 Prometheus 正常联通：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201601543.png" alt="image-20230420160147459" style="zoom:67%;"><p>点击 Back 返回即可，可以看到 Data Sources 页面，出现了添加的 Prometheus:</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201602307.png" alt="image-20230420160203226" style="zoom:80%;"><h2 id="创建仪表盘⭐" tabindex="-1">创建仪表盘⭐ <a class="header-anchor" href="#创建仪表盘⭐" aria-label="Permalink to &quot;创建仪表盘⭐&quot;">​</a></h2><blockquote><p>⾯板是属于某⼀个 Dashboard 的，所以我们需要先创建⼀个 Dashboard，在侧边栏点击 + 切换到Dashboard 下⾯开始创建 Dashboard：</p></blockquote><h3 id="仪表盘配置⭐" tabindex="-1">仪表盘配置⭐ <a class="header-anchor" href="#仪表盘配置⭐" aria-label="Permalink to &quot;仪表盘配置⭐&quot;">​</a></h3><h4 id="添加cpu面板" tabindex="-1">添加CPU面板 <a class="header-anchor" href="#添加cpu面板" aria-label="Permalink to &quot;添加CPU面板&quot;">​</a></h4><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261515276.png" alt="image-20230426151514180" style="zoom:80%;"><p>⽐如我们现在就要来查询节点的 CPU 使⽤率，前⾯在 node_exporter 章节中已经学习了该监控数据的查询语句为</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292101732.png" alt="image-20230429210119634" style="zoom:80%;"><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=</span><span style="color:#FFCB6B;">&quot;idle&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">))</span><span style="color:#A6ACCD;"> by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> ) </span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span></code></pre></div><p>只需要将该语句填充到查询的 PromQL 语句中即可在上⾯显示出监控的结果</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261516273.png" alt="image-20230426151607174" style="zoom:80%;"><p>点击右上⻆的 Apply 按钮即可创建成功⼀个 Panel ⾯板。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261516565.png" alt="image-20230426151633467" style="zoom:80%;"><h4 id="添加内存面板" tabindex="-1">添加内存面板 <a class="header-anchor" href="#添加内存面板" aria-label="Permalink to &quot;添加内存面板&quot;">​</a></h4><p>⽤同样的⽅式我们可以创建⼀个⽤于查询节点内存使⽤率的⾯板：</p><p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292103232.png" alt="image-20230429210327091"></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292104350.png" alt="image-20230429210400163" style="zoom:80%;"><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemTotal_bytes</span><span style="color:#89DDFF;">)*</span><span style="color:#A6ACCD;">100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261517048.png" alt="image-20230426151719947" style="zoom:80%;"><h4 id="排列位置" tabindex="-1">排列位置 <a class="header-anchor" href="#排列位置" aria-label="Permalink to &quot;排列位置&quot;">​</a></h4><p>创建完成后的⾯板我们也可以拖动他们的排列位置：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261517343.png" alt="image-20230426151737248" style="zoom:80%;"><p>如果还想重新编辑⾯板，可以点击标题，在弹出来的下拉框中选择 Edit 编辑即可</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261517843.png" alt="image-20230426151755741" style="zoom:80%;"><h3 id="下拉框配置" tabindex="-1">下拉框配置 <a class="header-anchor" href="#下拉框配置" aria-label="Permalink to &quot;下拉框配置&quot;">​</a></h3><blockquote><p>现在我们在⼀个 Dashboard 中添加了两个 Panel，我们可以很明显看到会直接将所有的节点信息展示在同⼀个⾯板中，但是如果有⾮常多的节点的话数据量就⾮常⼤了，这种情况下我们最好的⽅式是将节点当成参数，可以让⽤户⾃⼰去选择要查看哪⼀个节点的监控信息，要实现这个功能，我们就需要去添加⼀个以节点为参数的变量来去查询监控数据。就是实现如下下拉框</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292111613.png" alt="image-20230429211103542" style="zoom:80%;"><p>点击 Dashboard ⻚⾯右上⽅的 Dashboard settings 按钮，进⼊配置⻚⾯：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261518531.png" alt="image-20230426151842442" style="zoom:80%;"><p>在该 Settings ⻚⾯可以来对整个 Dashboard 进⾏配置，⽐如名称、标签、变量等：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261519421.png" alt="image-20230426151905323" style="zoom:80%;"><p>这⾥我们点击左边的 Variables 添加⼀个变量，变量⽀持更具交互性和动态性的仪表板，可以在它们的位置使⽤变量，⽽不是在指标查询中硬编码，变量显示为 Dashboard 顶部的下拉列表，这些下拉列表可以轻松更改仪表板中显示数据</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261519258.png" alt="image-20230426151937158" style="zoom:80%;"><p>为了能够选择节点数据，这⾥我们定义了⼀个名为 instance 的变量名，在添加变量的⻚⾯中主要包括如下⼀些属性：</p><blockquote><p>Name ：变量名，在仪表盘中调⽤使⽤ $变量名 的⽅式</p><p>Type ：变量类型，变量类型有多种，其中 query 表示这个变量是⼀个查询语句</p><p>Hide ：为空是表现为下拉框，选择 label 表示不显示下拉框的名字，选择 variable 表示隐藏该变量，该变量不会在 Dashboard 上⽅显示出来，默认选择为空</p><p>Data source ：查询语句的数据源Refresh ：何时去更新变量的值，变量的值是通过查询数据源获取到的，但是数据源本身也会发⽣变化，所以要时不时的去更新变量的值，这样数据源的改变才会在变量对应的下拉框中显示出来。</p><p>Refresh 有两个值可以选择： On Dashboard Load （在 Dashboard 加载时更新）、 On Time Range</p><p>Change （在时间范围变更的时候更新）</p><p>Query ：查询表达式，不同的数据源查询表达式都不同</p><p>Regex ：正则表达式，⽤来对抓取到的数据进⾏过滤，默认不过滤</p><p>Sort ：排序，对下拉框中的变量值做排序，默认是 disable，表示查询结果是怎样下拉框就怎样显示</p><p>Multi-value ：启⽤这个功能，变量的值就可以选择多个，具体表现在变量对应的下拉框中可以选多个值的组合</p><p>Include All option ：启⽤这个功能，变量下拉框中就多了⼀个全选 all 的选项</p></blockquote><p>我们还可以使⽤⼀个 label_values() 的函数来直接获取查询结果中的某个 label 标签的值：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261521531.png" alt="image-20230426152111431" style="zoom:80%;"><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">label_values(up{job=&quot;node-exporter&quot;},instance)</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261521321.png" alt="image-20230426152137215" style="zoom:67%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261521030.png" alt="image-20230426152149918" style="zoom:80%;"><p>回到 Dashboard ⻚⾯就可以看到多了⼀个 选择节点 的下拉框：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261522487.png" alt="image-20230426152208358" style="zoom:80%;"><p>但是这个时候的⾯板并不会随着我们下拉框的选择⽽变化，我们需要将 instance 这个变量传⼊查询语句中，⽐如重新修改 CPU使⽤率 的查询语句</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=</span><span style="color:#FFCB6B;">&quot;idle&quot;</span><span style="color:#FFCB6B;">,instance</span><span style="color:#A6ACCD;">=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">))</span><span style="color:#A6ACCD;"> by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> /</span></span>
<span class="line"><span style="color:#FFCB6B;">sum(rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#A6ACCD;">)) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> )</span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261523908.png" alt="image-20230426152302798" style="zoom:80%;"><p>以同样的⽅式修改内存使⽤率</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemAvailable_bytes{instance=~</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemTotal_bytes{instance=~</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">)*</span><span style="color:#A6ACCD;"> 100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261523297.png" alt="image-20230426152355206" style="zoom:80%;"><p>回到 Dashboard ⻚⾯就可以根据我们的下拉框来选择需要监控的节点数据了，定义参数的时候如果选择了可以选择所有，同样可以查看所有节点的数据</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261524677.png" alt="image-20230426152416586" style="zoom:80%;"><p>我们也可以勾选多选框</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261524478.png" alt="image-20230426152455374" style="zoom:67%;"><p>可以我们就可以勾选多个</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261525719.png" alt="image-20230426152510629" style="zoom:80%;"><p>多个查询</p><p>编辑 CPU 使⽤率这个⾯板，在⾯板编辑器下⽅的 Query 区域点击 + Query 按钮新增⼀个查询：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">sum(rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=</span><span style="color:#FFCB6B;">&quot;user&quot;</span><span style="color:#FFCB6B;">,instance</span><span style="color:#A6ACCD;">=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) by</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) by</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> ) </span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261525263.png" alt="image-20230426152547160" style="zoom:80%;"><p>在添加idle的使⽤率的表达式</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">(sum(rate(node_cpu_seconds_total{mode=&quot;idle&quot;,instance=~&quot;$instance&quot;}[5m])) by</span></span>
<span class="line"><span style="color:#A6ACCD;">(instance) / sum(rate(node_cpu_seconds_total{instance=~&quot;$instance&quot;}[5m])) by</span></span>
<span class="line"><span style="color:#A6ACCD;">(instance) ) * 100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261526946.png" alt="image-20230426152611844" style="zoom:80%;"><p>依次在添加iowait和system和其他的使⽤率</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">sum(rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=</span><span style="color:#FFCB6B;">&quot;iowait&quot;</span><span style="color:#FFCB6B;">,instance</span><span style="color:#A6ACCD;">=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">by</span><span style="color:#A6ACCD;"> (instance) / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">by</span><span style="color:#A6ACCD;"> (instance) ) </span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">sum(rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=</span><span style="color:#FFCB6B;">&quot;system&quot;</span><span style="color:#FFCB6B;">,instance</span><span style="color:#A6ACCD;">=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">by</span><span style="color:#A6ACCD;"> (instance) / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">)</span></span>
<span class="line"><span style="color:#FFCB6B;">by</span><span style="color:#A6ACCD;"> (instance) ) </span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#其他cpu使⽤</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">sum(rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode!~</span><span style="color:#FFCB6B;">&quot;system|user|iowait|idle&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) by</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) by</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> ) </span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261527494.png" alt="image-20230426152749327" style="zoom:80%;"><h3 id="图例配置" tabindex="-1">图例配置 <a class="header-anchor" href="#图例配置" aria-label="Permalink to &quot;图例配置&quot;">​</a></h3><blockquote><p>上⾯转换完成后，可以看到 Legend 部分展示的图例较多，我们可以将 Legend 的模式修改为 Table，此外我们还可以通过 Legend values 来选择⼀些其他的信息进⾏展示，⽐如最⼤值、最⼩值、平均值等等：</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261528774.png" alt="image-20230426152816660" style="zoom:80%;"><h3 id="其他选项" tabindex="-1">其他选项 <a class="header-anchor" href="#其他选项" aria-label="Permalink to &quot;其他选项&quot;">​</a></h3><h4 id="单位" tabindex="-1">单位 <a class="header-anchor" href="#单位" aria-label="Permalink to &quot;单位&quot;">​</a></h4><blockquote><p>由于我们这⾥计算的都是百分⽐，所以可以将单位设置为 % ，位于右侧的 Standard options 下⽅的Unit 中选择 Misc -&gt; Percent(0-100) 即可：</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261528928.png" alt="image-20230426152841818" style="zoom:80%;"><h4 id="图例样式" tabindex="-1">图例样式 <a class="header-anchor" href="#图例样式" aria-label="Permalink to &quot;图例样式&quot;">​</a></h4><p>此外还可以配置图例的最⼩值、最⼤值、保留⼩数的位数、图形颜⾊配置等等：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261530154.png" alt="image-20230426153014032" style="zoom:67%;"><h3 id="时间间隔⭐" tabindex="-1">时间间隔⭐ <a class="header-anchor" href="#时间间隔⭐" aria-label="Permalink to &quot;时间间隔⭐&quot;">​</a></h3><blockquote><p>前⾯我们在查询监控数据的时候都是将区间向量的范围固定成了 1m 或者 5m ，这样固定后显然不是⾮常灵活，所以我们可以再添加⼀个时间间隔的参数来灵活选择。</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292125926.png" alt="image-20230429212510839" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261530580.png" alt="image-20230426153037483" style="zoom:80%;"><blockquote><p>这⾥我们新增了⼀个名为 interval 的参数，不过需要注意该参数的类型为 Interval ，然后我们配置该参数可选的值包括 1m,5m,10m,30m ，添加后在 Dashboard ⻚⾯上就会多⼀个时间间隔的下拉框。然后记得将查询语句中的相关时间间隔替换成 $interval 参数：</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261530626.png" alt="image-20230426153058525" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261531648.png" alt="image-20230426153112549" style="zoom:80%;"><h3 id="覆盖配置⭐" tabindex="-1">覆盖配置⭐ <a class="header-anchor" href="#覆盖配置⭐" aria-label="Permalink to &quot;覆盖配置⭐&quot;">​</a></h3><p>⽤同样的⽅式可以对内存监控图表进⾏修改，⽐如在⼀个图表中展示总内存、已⽤内存、可⽤内存、内存使⽤率等信息</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 内存使用率</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_memory_MemTotal_bytes</span><span style="color:#89DDFF;">)*</span><span style="color:#A6ACCD;">100</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 新增总内存的查询</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemTotal_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 已⽤内存的查询</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemTotal_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}-node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 以及可⽤内存的查询</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span></span></code></pre></div><p>依次添加查询</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292143217.png" alt="image-20230429214318116" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292135432.png" alt="image-20230429213551331" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292136569.png" alt="image-20230429213604481" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292136563.png" alt="image-20230429213655467" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292137399.png" alt="image-20230429213736307" style="zoom:80%;"><p>如下图所示：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261534969.png" alt="image-20230426153428869" style="zoom:80%;"><p>我们可以先将整个⾯板的单位调整为 bytes(IEC) ，该形式的单位会⾃动在 GiB、MiB、KiB 之间进⾏换算</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261534294.png" alt="image-20230426153449188" style="zoom:80%;"><p>但是明显使⽤率的单位是不正确的，这个时候就需要我们针对该查询进⾏覆盖配置，点击编辑器右侧Overrides 选项卡进⾏覆盖配置：点击选项卡中的 + Add field override 按钮可以对属性进⾏覆盖操作</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261535850.png" alt="image-20230426153510752" style="zoom:80%;"><p>可选的⽅式有很多，⽐如可以设置指定具体属性的值，也可以根据正则表达式来匹配属性的名进⾏配置，还可以根据指定的类型以及指定的 query 查询结果进⾏覆盖，显然我们这⾥选择最后⼀项 Fieldsreturned by query</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261537407.png" alt="image-20230426153758290" style="zoom:80%;"><p>直接选择查询使⽤率的 A 这条语句进⾏覆盖</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261540201.png" alt="image-20230426154006094" style="zoom:80%;"><p>先加上百分⽐的单位</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261540250.png" alt="image-20230426154028101" style="zoom:80%;"><p>选择Misc--Percent（0-100）</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261540421.png" alt="image-20230426154042324" style="zoom:80%;"><p>同样还可以把百分⽐显示在图形右侧</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261540375.png" alt="image-20230426154058276" style="zoom:80%;"><p>这样我们就在同⼀个图形当中显示出了不同单位的两种图形</p><h2 id="表格面板" tabindex="-1">表格面板 <a class="header-anchor" href="#表格面板" aria-label="Permalink to &quot;表格面板&quot;">​</a></h2><blockquote><p>接下来我们以统计服务器资源为例，对表格⾯板的使⽤进⾏说明。⽐如通过表格展示Linux服务器总内存、总 CPU 数、带宽等信息，接下来分别添加如下查询：</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 总内存</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemTotal_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># CPU 核数</span></span>
<span class="line"><span style="color:#FFCB6B;">count(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">system</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;">) by</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 连接数</span></span>
<span class="line"><span style="color:#FFCB6B;">node_netstat_Tcp_CurrEstab</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 下⾏带宽，注意$interval是前面定义的时间变量，如果没定义，就写5m即可</span></span>
<span class="line"><span style="color:#FFCB6B;">max(rate(node_network_receive_bytes_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[$interval]</span><span style="color:#A6ACCD;">)</span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;">8) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 上⾏带宽 ，注意$interval是前面定义的时间变量，如果没定义，就写5m即可</span></span>
<span class="line"><span style="color:#FFCB6B;">max(rate(node_network_transmit_bytes_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[$interval]</span><span style="color:#A6ACCD;">)</span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;">8) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span></span></code></pre></div><p>不断增加查询即可</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292148554.png" alt="image-20230429214833446" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292148599.png" alt="image-20230429214859480" style="zoom:80%;"><p>后面也一样</p><p>如还有其他需要展示的也可以直接添加新的查询即可,在 Dashboard 中添加添加⼀个空的 Panel ⾯板，进⼊⾯板编辑器后在右侧上⽅选择 Table ⾯板：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261548247.png" alt="image-20230426154829154" style="zoom:80%;"><p>添加完成后如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261550421.png" alt="image-20230426155011317" style="zoom:80%;"><p>可以看到数据是以时间序列展示的，我们需要修改下，注意，每一个都要修改</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261550680.png" alt="image-20230426155025596" style="zoom:80%;"><p>当每个query都修改后，就变成了如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261550855.png" alt="image-20230426155042756" style="zoom:80%;"><h3 id="表格转换" tabindex="-1">表格转换 <a class="header-anchor" href="#表格转换" aria-label="Permalink to &quot;表格转换&quot;">​</a></h3><p>现在我们将所有需要展示的信息都通过 Table 形式展示出来了，但是现在有⼀个很⼤的问题，就是需要</p><p>对每⼀个查询切换显示：每⼀个query显示⼀个表格（因为有5个query，所以有5个表格可以选择）</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261551206.png" alt="image-20230426155103108" style="zoom:80%;"><p>这显然是⾮常不友好的显示⽅式，我们需要将这些表格内容合并成⼀个表格进⾏展示，这个时候就需要⽤到 Grafana 的 Transform 转换功能了，在 Transform 选项卡中选择 Merge 选项</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261552286.png" alt="image-20230426155242187" style="zoom:80%;"><p>Merge 转换器可以合并多个序列或者多个表格为⼀个表格，其中可合并的值将合并到同⼀⾏中，⽤于显示在表格中可视化的多个序列、表格或两者的组合。当我们选择了 Merge 转换器过后就会将上⾯的多个查询结果合并成⼀个表格。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261553642.png" alt="image-20230426155304544" style="zoom:80%;"><p>但其实合并后的表格数据有⼀些地⽅没有显示，这是因为我们查询的结果实际上是包含 <strong>name</strong> 这个标签的，在合并的时候会造成数据丢失，我们可以将所有的查询语句后⾯添加上⼀个 - 0 来去掉这个标签，这样合并的时候就不会丢失数据了。⽐如查询主机信息的语句变成 node_uname_info{job=&quot;node-exporter&quot;} - 0 ，其他的语句也都加上 - 0 这个操作。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261553934.png" alt="image-20230426155336842" style="zoom:80%;"><p>全部加上-0后，PromQL表达式变成如下：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 总内存</span></span>
<span class="line"><span style="color:#FFCB6B;">node_memory_MemTotal_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># CPU 核数</span></span>
<span class="line"><span style="color:#FFCB6B;">count(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mode=</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">system</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">}</span><span style="color:#A6ACCD;">) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> - 0</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 连接数</span></span>
<span class="line"><span style="color:#FFCB6B;">node_netstat_Tcp_CurrEstab</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">0</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 下载带宽 </span></span>
<span class="line"><span style="color:#FFCB6B;">max(rate(node_network_receive_bytes_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[$interval]</span><span style="color:#A6ACCD;">)</span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;">8) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> - 0</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 上传带宽 </span></span>
<span class="line"><span style="color:#FFCB6B;">max(rate(node_network_transmit_bytes_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[$interval]</span><span style="color:#A6ACCD;">)</span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;">8) by</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> - 0</span></span></code></pre></div><p>数据虽然正确了，但是表头却看不出来是表达的什么意义，有的列还是不需要的，这个时候同样也要⽤到Transform 转换器了，这⾥我们需要⽤到的是 Organize fields 转换器，该转换器允许⽤户重新排序、隐藏或重命名字段或者列。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261554272.png" alt="image-20230426155423173" style="zoom:80%;"><p>这⾥我们将不需要的⼀些列隐藏掉了，将表头进⾏了重命名。</p><h3 id="属性覆盖" tabindex="-1">属性覆盖 <a class="header-anchor" href="#属性覆盖" aria-label="Permalink to &quot;属性覆盖&quot;">​</a></h3><p>上⾯我们将查询的数据结果⽤⼀个表格展示出来了，但是现在我们的数据都是直接的⼀个结果，我们需要进⾏转换加上我们的单位。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261554354.png" alt="image-20230426155448268" style="zoom:80%;"><p>⽐如对于总内存这⼀列，我们需要添加覆盖，设置⼀个 bytes 类型的单位，然后会根据我们的结果进⾏⾃动转换：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261555554.png" alt="image-20230426155526452" style="zoom:80%;"><p>同样对下⾏和上⾏宽带这两列的数据结果进⾏覆盖，添加 bytes/sec(IEC) 类型的单位：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261555160.png" alt="image-20230426155548056" style="zoom:80%;"><h3 id="样式配置" tabindex="-1">样式配置 <a class="header-anchor" href="#样式配置" aria-label="Permalink to &quot;样式配置&quot;">​</a></h3><p>此外在定制某列数据的显示效果的时候我们还可以为其配置背景颜⾊，只需要设置 Cell Options&gt;Cell Type ，该属性可以配置⽂本颜⾊、背景等⽅式：如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261556327.png" alt="image-20230426155613219" style="zoom:80%;"><p>修改背景颜⾊为绿⾊：</p><p>Shandard options &gt; Color scheme 如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261556902.png" alt="image-20230426155648786" style="zoom:80%;"><h2 id="仪表盘和统计面板" tabindex="-1">仪表盘和统计⾯板 <a class="header-anchor" href="#仪表盘和统计面板" aria-label="Permalink to &quot;仪表盘和统计⾯板&quot;">​</a></h2><h3 id="仪表盘-2" tabindex="-1">仪表盘 <a class="header-anchor" href="#仪表盘-2" aria-label="Permalink to &quot;仪表盘&quot;">​</a></h3><blockquote><p>如果想展示与最⼤值和最⼩值相关的数据，我们可以选择使⽤仪表盘⾯板，⽐如我们⽤⼀个仪表盘⾯板来展示内存使⽤率。</p></blockquote><h4 id="标准仪表盘" tabindex="-1">标准仪表盘 <a class="header-anchor" href="#标准仪表盘" aria-label="Permalink to &quot;标准仪表盘&quot;">​</a></h4><p>在 Dashboard ⻚⾯上点击创建⼀个新的空⾯板，在⾯边编辑器右上⽅选择 Gauge 类型的⾯板：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261120999.png" alt="image-20230426112005911" style="zoom:80%;"><p>然后添加如下所示的查询语句，获取内存使⽤率：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;">{instance</span><span style="color:#89DDFF;">=</span><span style="color:#C3E88D;">~</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}/node_memory_MemTotal_</span></span>
<span class="line"><span style="color:#FFCB6B;">bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#89DDFF;">)*</span><span style="color:#A6ACCD;">100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261121092.png" alt="image-20230426112110019" style="zoom:80%;"><h4 id="条形仪表盘" tabindex="-1">条形仪表盘 <a class="header-anchor" href="#条形仪表盘" aria-label="Permalink to &quot;条形仪表盘&quot;">​</a></h4><p>除了上⾯标准的仪表盘⾯板之外，还有条形仪表盘，该⾯板可以显示⼀个或多个条形仪表，同样我们可以⽤来展示 CPU 使⽤率、内存使⽤率等。⽐如这⾥我们⽤条形仪表盘来进⾏展示。点击添加⼀个新的空⾯板，在⾯板编辑器右侧选择 Bar gauge ：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261406732.png" alt="image-20230426140650650" style="zoom:80%;"><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># cpu使⽤率promql表达式</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{mode=</span><span style="color:#FFCB6B;">&quot;idle&quot;</span><span style="color:#FFCB6B;">,instance</span><span style="color:#A6ACCD;">=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">))</span><span style="color:#A6ACCD;"> by</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;"> / sum</span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">rate(node_cpu_seconds_total</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}[5m]</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) by </span><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">instance</span><span style="color:#89DDFF;">)</span><span style="color:#A6ACCD;">) </span><span style="color:#89DDFF;">*</span><span style="color:#A6ACCD;"> 100</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 内存使⽤率promql表达式</span></span>
<span class="line"><span style="color:#89DDFF;">(</span><span style="color:#FFCB6B;">1</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-node_memory_MemAvailable_bytes</span><span style="color:#A6ACCD;">{instance</span><span style="color:#89DDFF;">=</span><span style="color:#C3E88D;">~</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">}/node_memory_MemTotal_b</span></span>
<span class="line"><span style="color:#FFCB6B;">ytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">)*</span><span style="color:#A6ACCD;">100</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261407303.png" alt="image-20230426140737732" style="zoom:80%;"><h3 id="统计面板" tabindex="-1">统计⾯板 <a class="header-anchor" href="#统计面板" aria-label="Permalink to &quot;统计⾯板&quot;">​</a></h3><p>统计⾯板可以⽤于显示⼀个⼤的统计值和⼀个可选的背景颜⾊，我们可以使⽤阈值来控制背景或颜⾊值，</p><p>效果如下所示：下⾯我们使⽤该⾯板来统计⼏个监控数据，⽐如节点运⾏时间、CPU 核数、总内存⼤⼩等等。</p><h4 id="文本模式" tabindex="-1">⽂本模式 <a class="header-anchor" href="#文本模式" aria-label="Permalink to &quot;⽂本模式&quot;">​</a></h4><p>⾸先创建⼀个空的⾯板，选择使⽤ Stat ⾯板：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261408417.png" alt="image-20230426140840333" style="zoom:80%;"><p>我们以显示系统运⾏时间为列：promql表达式为</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">avg(time(</span><span style="color:#A6ACCD;">) - node_boot_time_seconds{instance=~&quot;$instance&quot;})</span></span></code></pre></div><p>我们把设置单位为 seconds (s) ，并且在 Thresholds 根据设置的值展示不同颜⾊，⽐如这⾥我们设置阈值在200 和 300 秒的时候显示不同的颜⾊，因为运⾏时间超过了300s，所以当前显示指定的颜⾊。如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261409160.png" alt="image-20230426140936089" style="zoom:80%;"><p>以同样的⽅式，我们在添加总内存的⼤⼩：promql表达式：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">node_memory_MemTotal_bytes</span><span style="color:#A6ACCD;">{instance=~</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#A6ACCD;">$instance</span><span style="color:#FFCB6B;">&quot;</span><span style="color:#FFCB6B;">}</span></span></code></pre></div><p>完成后如下图：我们设置单位为 bytes(IEC) ，并且在 Thresholds 根据设置不同阀值展示不同颜⾊。因为内存总⼤⼩</p><p>当前的值超过了3，所以显示深蓝⾊</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261410075.png" alt="image-20230426141021996" style="zoom:80%;"><h4 id="背景模式" tabindex="-1">背景模式 <a class="header-anchor" href="#背景模式" aria-label="Permalink to &quot;背景模式&quot;">​</a></h4><p>可以在stat styles⾥⾯设置背景，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261412089.png" alt="image-20230426141212005" style="zoom:80%;"><h2 id="仪表盘市场⭐" tabindex="-1">仪表盘市场⭐ <a class="header-anchor" href="#仪表盘市场⭐" aria-label="Permalink to &quot;仪表盘市场⭐&quot;">​</a></h2><blockquote><p>Grafana的仪表盘市场下载一个Dashboard，市场地址：<a href="https://grafana.com/grafana/dashboards" target="_blank" rel="noreferrer">https://grafana.com/grafana/dashboards</a></p></blockquote><h3 id="搜索仪表盘" tabindex="-1">搜索仪表盘 <a class="header-anchor" href="#搜索仪表盘" aria-label="Permalink to &quot;搜索仪表盘&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304192034492.png" alt="image-20230419203425383" style="zoom:80%;"><blockquote><p>例如，搜索linux相关图表</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304192048764.png" alt="image-20230419204839647" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304192049421.png" alt="image-20230419204908345" style="zoom:80%;"><h3 id="导入仪表盘" tabindex="-1">导入仪表盘 <a class="header-anchor" href="#导入仪表盘" aria-label="Permalink to &quot;导入仪表盘&quot;">​</a></h3><blockquote><p>在 Grafana 里，仪表盘的配置可以通过图形化界面进行，但配置好的仪表盘是以 JSON 存储的。这也就是说，如果你把你的 JSON 数据分享出去，别人导入就可以直接导入同样的仪表盘（前提是你们的监测数据一样）。</p></blockquote><p>Grafana 的仪表盘市场：<a href="https://grafana.com/grafana/dashboards" target="_blank" rel="noreferrer">https://grafana.com/grafana/dashboards</a></p><p>比如说针对以下一些服务的标准仪表盘就可以在这里找到</p><p>那么，这里我们就用一个标准的仪表盘：<a href="https://grafana.com/grafana/dashboards/1860" target="_blank" rel="noreferrer">https://grafana.com/grafana/dashboards/1860</a></p><p>点击 Grafana 界面左侧 ”+”号，选择 import：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201607059.png" alt="image-20230420160727901" style="zoom:67%;"><p>上传 JSON 文件：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201607690.png" alt="image-20230420160747589" style="zoom:80%;"><p>配置模板信息：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304201608588.png" alt="image-20230420160804460" style="zoom:80%;"><p>导入完，在首页即可看见添加的仪表盘，点击进去查看：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304251107870.png" alt="image-20230425110719709" style="zoom:80%;"><h2 id="用户和角色" tabindex="-1">用户和角色 <a class="header-anchor" href="#用户和角色" aria-label="Permalink to &quot;用户和角色&quot;">​</a></h2><h3 id="新增新用户" tabindex="-1">新增新⽤户 <a class="header-anchor" href="#新增新用户" aria-label="Permalink to &quot;新增新⽤户&quot;">​</a></h3><p>Grafana 新增新⽤户分为两种⽅式：</p><blockquote><p>1，通过管理员账户邀请新⽤户，新⽤户通过邮箱或者浏览器修改其账户信息（需要提前开启，邮件功能）。</p><p>2，通过管理员账户⼿动创建账户。通过管理员账户⼿动创建账户，选择左边的防护墙图标，点击Users</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261542960.png" alt="image-20230426154228859" style="zoom:80%;"><p>出来的Server admin⽤户管理界⾯，添加右边的“new user”，创建新⽤户</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261542037.png" alt="image-20230426154245925" style="zoom:80%;"><p>填写新⽤户的名字，邮件，以及密码，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261543003.png" alt="image-20230426154302917" style="zoom:80%;"><p>指定⽤户权限</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261543102.png" alt="image-20230426154329998" style="zoom:80%;"><p>完成后可以在⽤户管理中查看到刚刚创建的新⽤户</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261543861.png" alt="image-20230426154343766" style="zoom:80%;"><h3 id="⻆色" tabindex="-1">⻆⾊ <a class="header-anchor" href="#⻆色" aria-label="Permalink to &quot;⻆⾊&quot;">​</a></h3><p>Grafana需要针对不同⽤户开放不同的 dashboard 权限，根据不同⻆⾊进⾏权限管理。Grafana ⻆⾊分为以下三类：</p><blockquote><p><strong>Admin Role（管理员⻆⾊）</strong>：<strong>可以做⼀切操作的组织。例如：添加和编辑数据源。添加和编辑组织⽤户和团队。配置App插件并设置组织设置</strong>。</p></blockquote><blockquote><p><strong>Editor Role（编辑⻆⾊）</strong>：<strong>可以创建和修改仪表板和警报规则。可以在特定⽂件夹和仪表板上禁⽤此功能。⽆法创建或编辑数据源，也⽆法邀请新⽤户</strong>。</p></blockquote><blockquote><p><strong>Viewer Role（查看者⻆⾊）</strong>：<strong>查看任何仪表板。可以在特定⽂件夹和仪表板上禁⽤此功能。⽆法创建或编辑仪表板或数据源</strong>。</p></blockquote><h2 id="告警功能" tabindex="-1">告警功能 <a class="header-anchor" href="#告警功能" aria-label="Permalink to &quot;告警功能&quot;">​</a></h2><blockquote><p>Grafana 除了支持丰富的数据源和图表功能外，还支持告警功能，该功能也使得 Grafana 从一个数据可视化工具成为了一个真正的监控利器。Grafana 可以通过 Alerting 模块的配置把监控数据中的异常信息进行告警，告警的规则可以直接基于现有的数据图表进行配置，在告警的时候也会把出现异常的图表进行通知，使告警通知更加友好。目前我自生产中使用Grafana的告警功能并不多。可能是因为我自己已经在Prometheus写好了告警规则。所以没必要了</p></blockquote><blockquote><p>1、有的同学觉得在Prometheus添加告警规则配置比较麻烦，你也可以在grafana里面来通过网页端添加</p><p>2、对于不想写脚本或开发语言的同学。又想对数据库的某些关键业务表进行监控（前提添加MySQL数据源），可以使用Grafanfa的告警功能，对数据库的某些重要的表做一个展示并监控报警。</p></blockquote><h3 id="修改配置文件" tabindex="-1">修改配置文件 <a class="header-anchor" href="#修改配置文件" aria-label="Permalink to &quot;修改配置文件&quot;">​</a></h3><blockquote><p>Grafana Alerting 支持多种告警渠道，比如Alertmanager、钉钉、微信、Discord、Email、Kafka、Pushover、Telegram、Webhook 等等，我们这里使用钉钉和 Email 进行展示说明。</p></blockquote><blockquote><p>Email配置：邮箱告警通常是最常见的告警接收方式，通过 Grafana 告警需要在 Grafana 的配置文件中配置 stmp 服务。在配置文件 /etc/grafana/grafana.ini 文件中添加 SMTP/Emailing 配置块并开启 Alerting ：</p></blockquote><p><a href="https://grafana.com/docs/grafana/latest/alerting/" target="_blank" rel="noreferrer">https://grafana.com/docs/grafana/latest/alerting/</a></p><h4 id="docker版本修改" tabindex="-1">Docker版本修改 <a class="header-anchor" href="#docker版本修改" aria-label="Permalink to &quot;Docker版本修改&quot;">​</a></h4><p>config.monitoring 配置一定要和我使用docker安装prometheus的方法一致：</p><p>注：因为当时docker安装没有把grafana.ini数据持久化，所以通过修改config.monitoring实现</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;&gt;</span><span style="color:#C3E88D;">grafana/config.monitoring</span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;">&quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">GF_SMTP_ENABLED=true</span></span>
<span class="line"><span style="color:#C3E88D;">GF_SMTP_HOST=smtp.163.com:465</span></span>
<span class="line"><span style="color:#C3E88D;">GF_SMTP_USER=cdring@163.com</span></span>
<span class="line"><span style="color:#C3E88D;">GF_SMTP_PASSWORD=MONBUFHOQZPBXHGD</span></span>
<span class="line"><span style="color:#C3E88D;">GF_SMTP_FROM_ADDRESS=cdring@163.com</span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker-compose</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span></span></code></pre></div><h4 id="二进制版本修改⭐" tabindex="-1">二进制版本修改⭐ <a class="header-anchor" href="#二进制版本修改⭐" aria-label="Permalink to &quot;二进制版本修改⭐&quot;">​</a></h4><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 修改grafana.ini</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/module/grafana-9.4.7/conf/defaults.ini</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">enabled</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">true</span></span>
<span class="line"><span style="color:#FFCB6B;">host</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">smtp.163.com:25</span></span>
<span class="line"><span style="color:#FFCB6B;">user</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">17315118673</span><span style="color:#C3E88D;">@163.com</span></span>
<span class="line"><span style="color:#FFCB6B;">password</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">HUSBWQHLTSIVPKOC</span></span>
<span class="line"><span style="color:#FFCB6B;">skip_verify</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">true</span></span>
<span class="line"><span style="color:#FFCB6B;">from_address</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">17315118673</span><span style="color:#C3E88D;">@163.com</span></span>
<span class="line"><span style="color:#FFCB6B;">from_name</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">=</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Grafana</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 重启Grafana</span></span>
<span class="line"><span style="color:#FFCB6B;">lsof</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-i:3000</span></span>
<span class="line"><span style="color:#82AAFF;">kill</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-9</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">43383</span></span>
<span class="line"><span style="color:#FFCB6B;">nohup</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./bin/grafana-server</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">web</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./grafana.log</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">2&gt;&amp;1</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&amp;</span></span></code></pre></div><h4 id="检查配置" tabindex="-1">检查配置 <a class="header-anchor" href="#检查配置" aria-label="Permalink to &quot;检查配置&quot;">​</a></h4><p><a href="http://192.168.11.61:3000/admin/settings" target="_blank" rel="noreferrer">http://192.168.11.61:3000/admin/settings</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292226205.png" alt="image-20230429222601113" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292226236.png" alt="image-20230429222647057" style="zoom:80%;"><p><strong>163邮箱开启pop3/smtp</strong></p><p>需要注意的是这里我们使用的是 163 的邮箱进行发送，在配置 smtp 的时候需要在邮箱中开启IMAP/SMTP 和 POP3/SMTP 两个服务，并添加一个授权码，上面的 password 密码使用的就是授权码进行认证：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261504269.png" alt="image-20230426150402141" style="zoom:80%;"><p><strong>修改默认邮件告警渠道</strong></p><p>回到 Grafana 页面中，点击左侧的Alerting在点击Contact points，在Contact point name修改grafana-default-email</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261504191.png" alt="image-20230426150449039" style="zoom:80%;"><p>修改接收报警信息的邮箱，测试没问题后，点保存</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261505391.png" alt="image-20230426150538283" style="zoom:80%;"><h3 id="其他告警渠道" tabindex="-1">其他告警渠道 <a class="header-anchor" href="#其他告警渠道" aria-label="Permalink to &quot;其他告警渠道&quot;">​</a></h3><h4 id="添加钉钉告警渠道" tabindex="-1">添加钉钉告警渠道 <a class="header-anchor" href="#添加钉钉告警渠道" aria-label="Permalink to &quot;添加钉钉告警渠道&quot;">​</a></h4><p>需要提前创建webhook地址和开放ip白名单，回到Contact points页面，点击new contact point，如下图</p><p><a href="https://oapi.dingtalk.com/robot/send?access_token=2ac0682516aa8634f3410c08339d21f7effeec5ac180eec60082a3ca66661f29" target="_blank" rel="noreferrer">https://oapi.dingtalk.com/robot/send?access_token=2ac0682516aa8634f3410c08339d21f7effeec5ac180eec60082a3ca66661f29</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261506160.png" alt="image-20230426150613068" style="zoom:80%;"><p>在Create contact point页面中，填写名称，以及选择对应对应的告警渠道，填写webhook地址，发送测试消息，完成后保存，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261506568.png" alt="image-20230426150632460" style="zoom:80%;"><h4 id="添加alertmanager告警渠道" tabindex="-1">添加alertmanager告警渠道 <a class="header-anchor" href="#添加alertmanager告警渠道" aria-label="Permalink to &quot;添加alertmanager告警渠道&quot;">​</a></h4><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261506695.png" alt="image-20230426150658602" style="zoom:80%;"><h4 id="配置默认接收的告警渠道" tabindex="-1">配置默认接收的告警渠道 <a class="header-anchor" href="#配置默认接收的告警渠道" aria-label="Permalink to &quot;配置默认接收的告警渠道&quot;">​</a></h4><p>点击左侧的Alerting在点击 Notification channels ，在default for all alerts默认中点编辑</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261507653.png" alt="image-20230426150719566" style="zoom:80%;"><p>编辑页面中，选择我们之前添加好的告警渠道。配置默认的告警渠道</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261507552.png" alt="image-20230426150735438" style="zoom:80%;"><h3 id="添加告警规则" tabindex="-1">添加告警规则 <a class="header-anchor" href="#添加告警规则" aria-label="Permalink to &quot;添加告警规则&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261508268.png" alt="image-20230426150844153" style="zoom:80%;"><p>告警规则填写：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292242926.png" alt="image-20230429224256820" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261509022.png" alt="image-20230426150907894" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261509980.png" alt="image-20230426150952858" style="zoom:80%;"><p>完成后，在alert rules的grafana显示新增加的告警规则，如下图：</p><p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292246530.png" alt="image-20230429224604420"></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261510865.png" alt="image-20230426151009773" style="zoom:80%;"><h3 id="测试" tabindex="-1">测试 <a class="header-anchor" href="#测试" aria-label="Permalink to &quot;测试&quot;">​</a></h3><p>把nginx关闭，然后测试邮件和钉钉是否收到告警消息，钉钉收到测试和mysqldown的告警信息如下</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">nginx</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292247244.png" alt="image-20230429224758139" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304292248526.png" alt="image-20230429224832427" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261511632.png" alt="image-20230426151102485" style="zoom:67%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261511410.png" alt="image-20230426151115303" style="zoom:80%;"><h1 id="alertmanager" tabindex="-1">Alertmanager <a class="header-anchor" href="#alertmanager" aria-label="Permalink to &quot;Alertmanager&quot;">​</a></h1><blockquote><p>Alertmanager是一个告警管理组件，Prometheus发出告警分为两步，首先，Prometheus按告警规则向Alertmanager发送警告，所以告警规则是在Prometheus上定义的，然后，由Alertmanager负责管理这些告警，比如把告警去重、分组、聚合等操作后通过邮件等方式将告警通知给对应的负责人。</p></blockquote><h2 id="概述-2" tabindex="-1">概述 <a class="header-anchor" href="#概述-2" aria-label="Permalink to &quot;概述&quot;">​</a></h2><p>告警能⼒在Prometheus的架构中被划分成两个独⽴的部分。如下所示，通过在Prometheus中定义AlertRule（告警规则），Prometheus会周期性的对告警规则进⾏计算，如果满⾜告警触发条件就会向Alertmanager发送告警信息。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262012112.png" alt="image-20230426201212000" style="zoom:80%;"><blockquote><p>Alertmanager作为⼀个独⽴的组件，负责接收并处理来⾃Prometheus Server(也可以是其它的客户端程序)的告警信息。Alertmanager可以对这些告警信息进⾏进⼀步的处理，⽐如当接收到⼤量重复告警时能够消除重复的告警信息，同时对告警信息进⾏分组并且路由到正确的通知⽅</p></blockquote><blockquote><p>Prometheus内置了对邮件，Slack等多种通知⽅式的⽀持，同时还⽀持与Webhook的集成，以⽀持更多定制化的场景。例如，⽬前还不⽀持钉钉，那⽤户完全可以通过Webhook与钉钉机器⼈进⾏集成，从⽽通过钉钉接收告警信息。同时AlertManager还提供了静默和告警抑制机制来对告警通知⾏为进⾏优化。</p></blockquote><p>Alertmanager除了提供基本的告警通知能⼒以外，还主要提供了如：分组、抑制以及静默等告警特性：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304262013328.png" alt="image-20230426201316214" style="zoom:80%;"><h3 id="分组" tabindex="-1">分组 <a class="header-anchor" href="#分组" aria-label="Permalink to &quot;分组&quot;">​</a></h3><blockquote><p>分组机制可以<strong>将详细的告警信息合并成⼀个通知</strong>。在某些情况下，⽐如由于系统宕机导致⼤量的告警被同时触发，在这种情况下分组机制可以将这些被触发的告警合并为⼀个告警通知，避免⼀次性接受⼤量的告警通知，⽽⽆法对问题进⾏快速定位。</p></blockquote><blockquote><p>例如，当集群中有数百个正在运⾏的服务实例，并且为每⼀个实例设置了告警规则。假如此时发⽣了⽹络故障，可能导致⼤量的服务实例⽆法连接到数据库，结果就会有数百个告警被发送到Alertmanager。⽽作为⽤户，可能只希望能够在⼀个通知中就能查看哪些服务实例收到影响。这时可以按照服务所在集群或者告警名称对告警进⾏分组，⽽将这些告警内聚在⼀起成为⼀个通知。告警分组，告警时间，以及告警的接受⽅式可以通过Alertmanager的配置⽂件进⾏配置</p></blockquote><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">default-receiver</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">30s</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">5m</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">repeat_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">4h</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">cluster</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">routes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">database-pager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match_re</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">service</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mysql|cassandra</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">frontend-pager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">product</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">environment</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">team</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">frontend</span></span></code></pre></div><blockquote><p>默认情况下所有的告警都会发送给集群管理员default-receiver，因此在Alertmanager的配置⽂件的根路由中，对告警信息按照集群以及告警的名称对告警进⾏分组。</p></blockquote><blockquote><p>如果告警时来源于数据库服务如MySQL或者Cassandra，此时则需要将告警发送给相应的数据库管理员(database-pager)。这⾥定义了⼀个单独⼦路由，如果告警中包含service标签，并且service为MySQL或者Cassandra,则向database-pager发送告警通知，由于这⾥没有定义group_by等属性，这些属性的配置信息将从上级路由继承，database-pager将会接收到按cluster和alertname进⾏分组的告警通知。</p></blockquote><blockquote><p>⽽某些告警规则来源可能来源于开发团队的定义，这些告警中通过添加标签team来标示这些告警的创建者。在Alertmanager配置⽂件的告警路由下，定义单独⼦路由⽤于处理这⼀类的告警通知，如果匹配到告警中包含标签team，并且team的值为frontend，Alertmanager将会按照标签product和environment对告警进⾏分组。此时如果应⽤出现异常，开发团队就能清楚的知道哪⼀个环境(environment)中的哪⼀个应⽤程序出现了问题，可以快速对应⽤进⾏问题定位。</p></blockquote><h3 id="抑制" tabindex="-1">抑制 <a class="header-anchor" href="#抑制" aria-label="Permalink to &quot;抑制&quot;">​</a></h3><blockquote><p>抑制是指<strong>当某⼀告警发出后，可以停⽌重复发送由此告警引发的其它告警的机制</strong>。例如，当集群不可访问时触发了⼀次告警，通过配置Alertmanager可以忽略与该集群有关的其它所有告警。这样可以避免接收到⼤量与实际问题⽆关的告警通知。抑制机制同样通过Alertmanager的配置⽂件进⾏设置。</p></blockquote><h3 id="配置文件" tabindex="-1">配置文件 <a class="header-anchor" href="#配置文件" aria-label="Permalink to &quot;配置文件&quot;">​</a></h3><p>Alertmanager主要负责对Prometheus产⽣的告警进⾏统⼀处理，因此在Alertmanager配置中⼀般会包含以下⼏个部分</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270933972.png" alt="image-20230427093330809" style="zoom:80%;"><blockquote><p>在全局配置中需要注意的是 resolve_timeout ，该参数定义了当Alertmanager持续多⻓时间未接收到告警后标记告警状态为resolved（已解决）。该参数的定义可能会影响到告警恢复通知的接收时间，读者可根据⾃⼰的实际场景进⾏定义，其默认值为5分钟。在接下来的部分，我们将已⼀些实际的例⼦解释Alertmanager的其它配置内容。</p></blockquote><h2 id="前置准备" tabindex="-1">前置准备 <a class="header-anchor" href="#前置准备" aria-label="Permalink to &quot;前置准备&quot;">​</a></h2><blockquote><p>Prometheus把产⽣的告警发送给Alertmanager进⾏告警处理时，需要在Prometheus使⽤的配置⽂件中添加关联Alertmanager组件的对应配置内容。</p></blockquote><h3 id="prometheus-yml" tabindex="-1">prometheus.yml <a class="header-anchor" href="#prometheus-yml" aria-label="Permalink to &quot;prometheus.yml&quot;">​</a></h3><p>1）编辑prometheus.yml⽂件加⼊关联Alertmanager组件的访问地址，如下</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># Alertmanager 配置</span></span>
<span class="line"><span style="color:#F07178;">alerting</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">alertmanagers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><p>2）设置rules文件位置</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">rule_files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">rules/*.yml</span><span style="color:#89DDFF;">&quot;</span></span></code></pre></div><p>3）添加监控Alertmanager，让Prometheus去收集Alertmanager的监控指标。</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">    </span><span style="color:#676E95;font-style:italic;"># 覆盖全局默认值，每15秒从该作业中刮取⼀次⽬标</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9090/-/reload</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270936316.png" alt="image-20230427093638145" style="zoom:80%;"><h3 id="rules-linux-yml" tabindex="-1">rules/linux.yml <a class="header-anchor" href="#rules-linux-yml" aria-label="Permalink to &quot;rules/linux.yml&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 进⼊到prometheus安装⽬录</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/prometheus</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus/alert.yml</span></span></code></pre></div><p>告警规则配置如下：</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">groups</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus alert</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">alert</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">服务告警</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">expr</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up{instance=&quot;node1:9100&quot;, job=&quot;linux监控&quot;} == 0</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">for</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">1m</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">critical</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">annotations</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#F07178;">summary</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">服务异常,实例:{{ $labels.instance }}</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#F07178;">description</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">{{ $labels.job }} 服务已关闭</span><span style="color:#89DDFF;">&quot;</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 检查规则写的是否正确</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/module/prometheus-2.29.1</span></span>
<span class="line"><span style="color:#FFCB6B;">./promtool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">check</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">rules</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">rules/linux.yml</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://node1:9090/-/reload</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://node1:9093/-/reload</span></span></code></pre></div><p>在告警规则⽂件中，我们可以将⼀组相关的规则设置定义在⼀个group下。在每⼀个group中我们可以定义多个告警规则(rule)。⼀条告警规则主要由以下⼏部分组成：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">alert：告警规则的名称。</span></span>
<span class="line"><span style="color:#A6ACCD;"></span></span>
<span class="line"><span style="color:#A6ACCD;">expr：基于PromQL表达式告警触发条件，⽤于计算是否有时间序列满⾜该条件。</span></span>
<span class="line"><span style="color:#A6ACCD;"></span></span>
<span class="line"><span style="color:#A6ACCD;">for：评估等待时间，可选参数。⽤于表示只有当触发条件持续⼀段时间后才发送告警。在等待期间新产⽣告警的状态为pending</span></span>
<span class="line"><span style="color:#A6ACCD;"></span></span>
<span class="line"><span style="color:#A6ACCD;">labels：⾃定义标签，允许⽤户指定要附加到告警上的⼀组附加标签。</span></span>
<span class="line"><span style="color:#A6ACCD;"></span></span>
<span class="line"><span style="color:#A6ACCD;">annotations：⽤于指定⼀组附加信息，⽐如⽤于描述告警详细信息的⽂字等，annotations的内容在告警产⽣时会⼀同作为参数发送到Alertmanager。</span></span></code></pre></div><h3 id="查看告警状态" tabindex="-1">查看告警状态 <a class="header-anchor" href="#查看告警状态" aria-label="Permalink to &quot;查看告警状态&quot;">​</a></h3><blockquote><p>重启Prometheus后，⽤户可以通过Prometheus WEB界⾯中的Alerts菜单查看当前Prometheus下的所有告警规则，以及其当前所处的活动状态。同时对于已经pending或者firing的告警，Prometheus也会将它们存储到时间序列ALERTS{}中。可以通过表达式，查询告警实例：</p></blockquote><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">ALERTS{}</span></span></code></pre></div><p>样本值为1表示当前告警处于活动状态（pending或者firing），当告警从活动状态转换为⾮活动状态时，样本值则为0</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301303197.png" alt="image-20230430130308026" style="zoom:80%;"><h3 id="测试告警规则" tabindex="-1">测试告警规则 <a class="header-anchor" href="#测试告警规则" aria-label="Permalink to &quot;测试告警规则&quot;">​</a></h3><blockquote><p>分别停止和启动node_exporter，这是监控linux系统的，查看状态</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span></code></pre></div><p>停止node-exporter，查看状态</p><blockquote><p>Prometheus⾸次检测到满⾜触发条件后，由于告警规则中设置了1分钟（for: 1m）的等待时间，告警状态从 INACTIVE 变为 Pending ，如下图所示：</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270946529.png" alt="image-20230427094633367" style="zoom:80%;"><p>如果1分钟后告警条件持续满⾜，告警状态从Penging变为FIRING，并且会把告警信息发送给alertmanager 。如下图</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270946226.png" alt="image-20230427094658052" style="zoom:80%;"><p>注：如果for: 0或者没有配置for，检测到满⾜触发条件，那告警状态从 INACTIVE 变为 FIRING ,把告警信息发送给 alertmanager，alertmanager接收到告警信息如下图：<a href="http://node1:9093/#/alerts" target="_blank" rel="noreferrer">http://node1:9093/#/alerts</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270947581.png" alt="image-20230427094728410" style="zoom:80%;"><h2 id="邮箱告警" tabindex="-1">邮箱告警 <a class="header-anchor" href="#邮箱告警" aria-label="Permalink to &quot;邮箱告警&quot;">​</a></h2><h3 id="配置邮箱服务" tabindex="-1">配置邮箱服务 <a class="header-anchor" href="#配置邮箱服务" aria-label="Permalink to &quot;配置邮箱服务&quot;">​</a></h3><p>获取邮箱授权码并开启<strong>smtp</strong>服务，<a href="https://mail.163.com/" target="_blank" rel="noreferrer">https://mail.163.com/</a> 登陆⾃⼰的163账号</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271005013.png" alt="image-20230427100506820" style="zoom:80%;"><p>Pop3/smtp/imap服务，点开启。弹出来扫描⼆维码点框。⼿机扫描发送短信。获取授权码</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271005986.png" alt="image-20230427100525808" style="zoom:80%;"><p>弹出来的框，复制授权码。就是⽤于第三⽅登录163邮箱的密码：MONBUFHOQZPBXHGD</p><h3 id="修改alertmanager配置" tabindex="-1">修改alertmanager配置 <a class="header-anchor" href="#修改alertmanager配置" aria-label="Permalink to &quot;修改alertmanager配置&quot;">​</a></h3><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 全局配置，配置发送邮件的信息</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_smarthost</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">smtp.163.com:25</span><span style="color:#A6ACCD;">       </span><span style="color:#676E95;font-style:italic;"># 腾讯邮箱服务器及端口</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_from</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">17315118673@163.com</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 用于发送告警的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_auth_username</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">17315118673@163.com</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 登录的邮箱名</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_auth_password</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">HUSBWQHLTSIVPKOC</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 邮箱授权码，请登录用于发送的邮箱在设置→账户中开启</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_require_tls</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">false</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 不启用TLS</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 路由组配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 分组依据，可分多个组</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">30s</span><span style="color:#A6ACCD;">        </span><span style="color:#676E95;font-style:italic;"># 第一次满足告警条件后等待多久发送邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">50s</span><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;"># 出现新告警后等待多久发送邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">repeat_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">2h</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 若已经成功发送邮件等待多久后再次发送</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">email</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">      </span><span style="color:#676E95;font-style:italic;"># 定义收件组名称</span></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 收件组配置</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">email</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">1597374863@qq.com</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 收件人邮箱，可添加多个邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#F07178;">inhibit_rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">critical</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">target_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">warning</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">equal</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dev</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">instance</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># altermanager</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://node1:9093/-/reload</span></span></code></pre></div><p>检查：<a href="http://192.168.11.61:9093/#/status" target="_blank" rel="noreferrer">http://192.168.11.61:9093/#/status</a></p><h3 id="配置触发器" tabindex="-1">配置触发器 <a class="header-anchor" href="#配置触发器" aria-label="Permalink to &quot;配置触发器&quot;">​</a></h3><p>查看现有触发器</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat prometheus/alert.yml</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 增加如下内容</span></span>
<span class="line"><span style="color:#F07178;">groups</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus alert</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">alert</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">服务告警</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">expr</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up{instance=&quot;node1:9100&quot;, job=&quot;linux监控&quot;} == 0</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">for</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">30s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">critical</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">annotations</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">{{ $labels.instance }}</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#F07178;">description</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">{{ $labels.job }} 服务已关闭</span><span style="color:#89DDFF;">&quot;</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://node1:9090/-/reload</span></span></code></pre></div><p>访问告警模块的<strong>web</strong>⻚⾯：<a href="http://192.168.11.61:9090/alerts" target="_blank" rel="noreferrer">http://192.168.11.61:9090/alerts</a></p><blockquote><p>INACTIVE：活跃中，即表示正常⽆告警产⽣。</p><p>PENDING：待触发，表示已经达到预设的阈值，但没达到预设的时间。</p><p>FIRING：表示达到预设的阈值并超过预设的时间触发告警</p></blockquote><h3 id="测试告警是否正常" tabindex="-1">测试告警是否正常 <a class="header-anchor" href="#测试告警是否正常" aria-label="Permalink to &quot;测试告警是否正常&quot;">​</a></h3><p>停⽌<strong>node-exporter</strong></p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># docker环境</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># ⼆进制安装环境</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span></code></pre></div><p><strong>prometheus</strong>查看</p><p>浏览器打开prometheus web管理⻚⾯--点击 Alerts,</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271013738.png" alt="image-20230427101308564" style="zoom:80%;"><p>检查<strong>alertmanager</strong>的<strong>alerts</strong>：<a href="http://192.168.11.61:9093/#/alerts" target="_blank" rel="noreferrer">http://192.168.11.61:9093/#/alerts</a></p><p>检查邮箱，登陆163邮箱查看邮件</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271013280.png" alt="image-20230427101345103" style="zoom:80%;"><p>如果没收到钉钉消息，排查</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">logs</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span></span></code></pre></div><h3 id="使用模板-非必须" tabindex="-1">使用模板(非必须) <a class="header-anchor" href="#使用模板-非必须" aria-label="Permalink to &quot;使用模板(非必须)&quot;">​</a></h3><p>看需求--不使⽤模版默认也⾏。</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/module/alertmanager-0.23.0</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 创建存放模版的⽬录</span></span>
<span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">template</span></span></code></pre></div><p>通过cat创建</p><div class="language-yaml"><button title="Copy Code" class="copy"></button><span class="lang">yaml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat &gt; template/email.tmpl &lt;&lt;EOF</span></span>
<span class="line"><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">define &quot;email.html&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span></span>
<span class="line"><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;">- </span><span style="color:#C3E88D;">if gt (len .Alerts.Firing) 0 -</span><span style="color:#89DDFF;">}}{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">range .Alerts</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">&lt;h2&gt;@告警通知&lt;/h2&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警程序</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus_alert &lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警级别</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Labels.severity</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">级 &lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警类型</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Labels.alertname</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">故障主机</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Labels.instance</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警主题</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Annotations.summary</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警详情</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Annotations.description</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">触发时间</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.StartsAt.Local.Format &quot;2006-01-02 15:04:05&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">end</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">end -</span><span style="color:#89DDFF;">}}</span></span>
<span class="line"><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;">- </span><span style="color:#C3E88D;">if gt (len .Alerts.Resolved) 0 -</span><span style="color:#89DDFF;">}}{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">range .Alerts</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">&lt;h2&gt;@告警恢复&lt;/h2&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警程序</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus_alert &lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">故障主机</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Labels.instance</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">故障主题</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Annotations.summary</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警详情</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.Annotations.description</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">告警时间</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.StartsAt.Local.Format &quot;2006-01-02 15:04:05&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#F07178;">恢复时间</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.EndsAt.Local.Format &quot;2006-01-02 15:04:05&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span><span style="color:#C3E88D;">&lt;br&gt;</span></span>
<span class="line"><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">end</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}{{</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">end -</span><span style="color:#89DDFF;">}}</span></span>
<span class="line"><span style="color:#89DDFF;">{{</span><span style="color:#A6ACCD;">- </span><span style="color:#C3E88D;">end</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">}}</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 检查看写的对不对</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager/template/email.tmpl</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 修改alertmanager配置,增加html这⾏</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager.yml</span></span></code></pre></div><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 模版配置</span></span>
<span class="line"><span style="color:#F07178;">templates</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">/opt/module/alertmanager-0.23.0/template/*.tmpl</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">email</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">    </span><span style="color:#676E95;font-style:italic;"># 收邮件的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">        </span><span style="color:#676E95;font-style:italic;"># 发送邮件的内容（调用模板文件中的）</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">html</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">{{ template &quot;email.html&quot; .}}</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 重载配置</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/-/reload</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 检查</span></span>
<span class="line"><span style="color:#FFCB6B;">http://192.168.88.101:9093/#/status</span></span></code></pre></div><p>163邮件报警</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301437789.png" alt="image-20230430143701687" style="zoom:80%;"><p>修改前和修改后的区别如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271019992.png" alt="image-20230427101906816" style="zoom:80%;"><h2 id="手机电话-短信告警" tabindex="-1">⼿机电话+短信告警 <a class="header-anchor" href="#手机电话-短信告警" aria-label="Permalink to &quot;⼿机电话+短信告警&quot;">​</a></h2><h3 id="安装步骤" tabindex="-1">安装步骤 <a class="header-anchor" href="#安装步骤" aria-label="Permalink to &quot;安装步骤&quot;">​</a></h3><p>睿象云（简称<strong>CA</strong>）安装步骤：<a href="https://www.aiops.com/" target="_blank" rel="noreferrer">https://www.aiops.com/</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261725852.png" alt="image-20230426172556728" style="zoom:80%;"><p>注册成功后并登陆，点击智能告警平台</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261934056.png" alt="image-20230426193405912" style="zoom:80%;"><h3 id="创建应用" tabindex="-1">创建应⽤ <a class="header-anchor" href="#创建应用" aria-label="Permalink to &quot;创建应⽤&quot;">​</a></h3><p>在点“集成”，然后找到“prometheus” 点下⾯的“+” 号</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261934570.png" alt="image-20230426193427432" style="zoom:80%;"><p>填写应⽤名称，在点保持并获取key</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261934085.png" alt="image-20230426193445924" style="zoom:80%;"><p>复制AppKey：2947c853c7c6430e8e87aa75f94304fa</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261936819.png" alt="image-20230426193606647" style="zoom:80%;"><h3 id="分派策略" tabindex="-1">分派策略 <a class="header-anchor" href="#分派策略" aria-label="Permalink to &quot;分派策略&quot;">​</a></h3><p>1、点击“配置”---2、“分派策略”---3、“新建分派”--4、填写“分派策略名称”---5、选择“分派条件”---6、选择“分派⼈”—7、点击“保存” 如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261936282.png" alt="image-20230426193630130" style="zoom:80%;"><h3 id="配置通知策略" tabindex="-1">配置通知策略 <a class="header-anchor" href="#配置通知策略" aria-label="Permalink to &quot;配置通知策略&quot;">​</a></h3><p>发⽣时：zabbix发⽣报警调⽤CA时触发，认领时：你收到短信后要回复认领时触发</p><p>关闭时：把收到的告警关闭时触发</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261936136.png" alt="image-20230426193658969" style="zoom:80%;"><p>如果配置通知策略有问题 点操作列 修改或者删除</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261937534.png" alt="image-20230426193714382" style="zoom:80%;"><h3 id="集成prometheus" tabindex="-1">集成prometheus <a class="header-anchor" href="#集成prometheus" aria-label="Permalink to &quot;集成prometheus&quot;">​</a></h3><p>修改<strong>alertmanager</strong>的配置⽂件，cd0ffd35ec9a4ac88dc944dc470d603b</p><p>通过webhook⽅式通知Cloud Alert，编辑告警的配置⽂件，新增以下内容。</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">vim alertmanager.yml</span></span>
<span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># 全局报警组，这个参数是必选的</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">ca</span></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">ca</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">webhook_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">url</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://api.aiops.com/alert/api/event/prometheus/2947c853c7c6430e8e87aa75f94304fa</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 重载alertmanager配置</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/-/reload</span></span></code></pre></div><h3 id="测试-1" tabindex="-1">测试 <a class="header-anchor" href="#测试-1" aria-label="Permalink to &quot;测试&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 关闭node-exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># docker环境</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># ⼆进制安装环境</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看prometheus的alerts: http://192.168.11.61:9090/alerts</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看alertmanager的alerts：http://192.168.11.61:9093/#/alerts</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 如果没收到消息，排查</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">logs</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span></span></code></pre></div><p>⼿机正常收到短信和电话报警信息如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261939621.png" alt="image-20230426193939447" style="zoom:80%;"><h2 id="企业钉钉告警" tabindex="-1">企业钉钉告警 <a class="header-anchor" href="#企业钉钉告警" aria-label="Permalink to &quot;企业钉钉告警&quot;">​</a></h2><h3 id="注册企业钉钉" tabindex="-1">注册企业钉钉 <a class="header-anchor" href="#注册企业钉钉" aria-label="Permalink to &quot;注册企业钉钉&quot;">​</a></h3><p>浏览器打开<a href="https://oa.dingtalk.com/register_new.htm?source=CreateTeamPC&amp;lwfrom=2019111111293551000#/" target="_blank" rel="noreferrer">钉钉注册⻚⾯</a> 填⼊⼿机号码，填⼊获取到的验证码，点注册</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261628750.png" alt="image-20230426162814592" style="zoom:80%;"><p>填⼊企业资料并注册</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261628202.png" alt="image-20230426162845084" style="zoom:80%;"><p>注册成功后，扫描⼆维码下载钉钉，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261629865.png" alt="image-20230426162916755" style="zoom:80%;"><h3 id="添加钉钉机器人" tabindex="-1">添加钉钉机器⼈ <a class="header-anchor" href="#添加钉钉机器人" aria-label="Permalink to &quot;添加钉钉机器⼈&quot;">​</a></h3><p>因为机器⼈添加，只能是钉钉电脑版（⼿机版钉钉不能添加机器⼈）。“测试钉钉报警“ 这个企业只有我⼀个⼈，所以我就把报警消息发到默认的 ”测试钉钉报警 全员群“ ⾥⾯。实际使⽤时，请创建个运维群--添加对应的⼈员进来。</p><p>电脑钉钉登陆成功后----点击左下⻆的。。。---然后再点管理后台，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261629199.png" alt="image-20230426162951081" style="zoom:80%;"><p>点击之前创建的企业名</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261630909.png" alt="image-20230426163012802" style="zoom:80%;"><p>点通讯录--组织架构--添加⼦部⻔</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261630847.png" alt="image-20230426163030739" style="zoom:80%;"><p>刷新后，把接收告警的员⼯调整到刚刚添加的部⻔</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261630616.png" alt="image-20230426163051507" style="zoom:80%;"><p>勾选部⻔</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261631971.png" alt="image-20230426163109859" style="zoom:80%;"><p>添加成功后，电脑钉钉消息窗⼝--会弹出⼀个测试告警的群--点击这个群---群设置--机器⼈</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261631196.png" alt="image-20230426163128076" style="zoom:80%;"><p>添加机器⼈</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261631919.png" alt="image-20230426163151811" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261631133.png" alt="image-20230426163159028" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261632304.png" alt="image-20230426163212190" style="zoom:80%;"><p>机器⼈名字：随意</p><p>勾选ip：填⼊alertmanager外⽹ip</p><p>如果在⾃⼰电脑上测试钉钉告警，获取Alertmanager外⽹ip地址的⽅法为浏览器打开<a href="http://ip138.com" target="_blank" rel="noreferrer">http://ip138.com</a> 您</p><p>的iP地址是：[xxx.xx.xx.xx]得到⾃⼰外⽹ip地址。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261634389.png" alt="image-20230426163409282" style="zoom:80%;"><p>复制机器⼈<strong>webhook</strong>地址</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261634886.png" alt="image-20230426163432766" style="zoom:80%;"><p>复制webhook的地址，如下：</p><p><a href="https://oapi.dingtalk.com/robot/send?access_token=2a89e1b607ff295d3347baf140e4b5ada2733ea852f1c51f79782b864b697a5f" target="_blank" rel="noreferrer">https://oapi.dingtalk.com/robot/send?access_token=2a89e1b607ff295d3347baf140e4b5ada2733ea852f1c51f79782b864b697a5f</a></p><p>我们真实需要的是access_token=后⾯的，2a89e1b607ff295d3347baf140e4b5ada2733ea852f1c51f79782b864b697a5f</p><h3 id="实现钉钉告警" tabindex="-1">实现钉钉告警 <a class="header-anchor" href="#实现钉钉告警" aria-label="Permalink to &quot;实现钉钉告警&quot;">​</a></h3><p><a href="https://github.com/timonwong/prometheus-webhook-dingtalk/releases" target="_blank" rel="noreferrer">https://github.com/timonwong/prometheus-webhook-dingtalk/releases</a></p><h4 id="二进制安装" tabindex="-1">二进制安装 <a class="header-anchor" href="#二进制安装" aria-label="Permalink to &quot;二进制安装&quot;">​</a></h4><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;">#安装⼆进制安装包</span></span>
<span class="line"><span style="color:#FFCB6B;">wget</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v2.1.0/prometheus-webhook-dingtalk-2.1.0.linux￾amd64.tar.gz</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#解压</span></span>
<span class="line"><span style="color:#FFCB6B;">tar</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">vxf</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk-2.1.0.linux-amd64.tar.gz</span></span>
<span class="line"><span style="color:#FFCB6B;">ls</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-l</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#移动并改名</span></span>
<span class="line"><span style="color:#FFCB6B;">mv</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk-2.1.0.linux-amd64</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/prometheus-webhook-dingtalk</span></span></code></pre></div><p>创建配置⽂件</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">cat &gt; /opt/prometheus-webhook-dingtalk/config.yml &lt;&lt;&quot;EOF&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">targets:</span></span>
<span class="line"><span style="color:#A6ACCD;">  webhook1:</span></span>
<span class="line"><span style="color:#A6ACCD;">    url: https://oapi.dingtalk.com/robot/send?access_token=修改为⾃⼰的TOKEN</span></span>
<span class="line"><span style="color:#A6ACCD;">    secret: SEC000000000000000000000</span></span>
<span class="line"><span style="color:#A6ACCD;">EOF</span></span></code></pre></div><p>创建⼀个 prometheus ⽤户：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">useradd -M -s /usr/sbin/nologin prometheus</span></span>
<span class="line"><span style="color:#A6ACCD;">chown prometheus:prometheus -R /opt/prometheus-webhook-dingtalk</span></span></code></pre></div><p>创建 systemd 服务</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/systemd/system/prometheus-webhook-dingtalk.service</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;"> &quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">[Unit]</span></span>
<span class="line"><span style="color:#C3E88D;">Description=prometheus-webhook-dingtalk</span></span>
<span class="line"><span style="color:#C3E88D;">Documentation=https://github.com/timonwong/prometheus-webhook-dingtalk</span></span>
<span class="line"><span style="color:#C3E88D;">[Service]</span></span>
<span class="line"><span style="color:#C3E88D;">User=prometheus</span></span>
<span class="line"><span style="color:#C3E88D;">Group=prometheus</span></span>
<span class="line"><span style="color:#C3E88D;">Restart=on-failure</span></span>
<span class="line"><span style="color:#C3E88D;">ExecStart=/opt/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk \</span></span>
<span class="line"><span style="color:#C3E88D;"> --config.file=/opt/prometheus-webhook-dingtalk/config.yml</span></span>
<span class="line"><span style="color:#C3E88D;">[Install]</span></span>
<span class="line"><span style="color:#C3E88D;">WantedBy=multi-user.target</span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 启动 prometheus-webhook-dingtalk</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">daemon-reload</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">start</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk.service</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 加⼊到开机⾃启动</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">enable</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk.service</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 检查</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">status</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk.service</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看 prometheus-webhook-dingtalk的⽇志以进⾏故障排除：</span></span>
<span class="line"><span style="color:#FFCB6B;">journalctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-u</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk.service</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span></span></code></pre></div><h4 id="docker安装⭐" tabindex="-1"><strong>Docker</strong>安装⭐ <a class="header-anchor" href="#docker安装⭐" aria-label="Permalink to &quot;**Docker**安装⭐&quot;">​</a></h4><p>创建配置⽂件<strong>config.yml</strong></p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 创建数据⽬录</span></span>
<span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus/prometheus-webhook-dingtalk/</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-p</span></span>
<span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus/prometheus-webhook-dingtalk/config.yml</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;">&quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">targets:</span></span>
<span class="line"><span style="color:#C3E88D;">  webhook1:</span></span>
<span class="line"><span style="color:#C3E88D;">    url: https://oapi.dingtalk.com/robot/send?access_token=2a89e1b607ff295d3347baf140e4b5ada2733ea852f1c51f79782b864b697a5f</span></span>
<span class="line"><span style="color:#C3E88D;">    secret: SEC000000000000000000000</span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><p><strong>docker-compose.yaml</strong>⽂件</p><p>注：我把prometheus-webhook-dingtalk安装在prometheus服务器上，如果安装在其他机器上也是可以的。</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cd /data/docker-prometheus/prometheus-webhook-dingtalk/</span></span>
<span class="line"><span style="color:#C3E88D;">cat &gt; docker-compose.yaml &lt;&lt; &quot;EOF&quot;</span></span>
<span class="line"><span style="color:#F07178;">version</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">3.3</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">webhook</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">timonwong/prometheus-webhook-dingtalk:v2.1.0</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">always</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">8060:8060</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--config.file=/etc/prometheus-webhook-dingtalk/config.yml</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">volumes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">./config.yml:/etc/prometheus-webhook-dingtalk/config.yml</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/etc/localtime:/etc/localtime:ro</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker-compose</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">ps</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">logs</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-dingtalk</span></span></code></pre></div><h3 id="访问地址" tabindex="-1">访问地址 <a class="header-anchor" href="#访问地址" aria-label="Permalink to &quot;访问地址&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261646628.png" alt="image-20230426164611530" style="zoom:80%;"><h3 id="alertmanager配置" tabindex="-1">alertmanager配置 <a class="header-anchor" href="#alertmanager配置" aria-label="Permalink to &quot;alertmanager配置&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">route:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 路由组配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">group_by:</span><span style="color:#A6ACCD;"> [</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">] </span><span style="color:#676E95;font-style:italic;"># 分组依据，可分多个组</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">group_wait:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">10</span><span style="color:#C3E88D;">s</span><span style="color:#A6ACCD;">        </span><span style="color:#676E95;font-style:italic;"># 第一次满足告警条件后等待多久发送邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">group_interval:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">10</span><span style="color:#C3E88D;">s</span><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;"># 出现新告警后等待多久发送邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">repeat_interval:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#C3E88D;">h</span><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;"># 若已经成功发送邮件等待多久后再次发送</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">receiver:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dingtalk</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">   </span><span style="color:#676E95;font-style:italic;"># 修改成钉钉</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#FFCB6B;">receivers:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 收件组配置</span></span>
<span class="line"><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">name:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">email</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">email_configs:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">to:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1597374863</span><span style="color:#C3E88D;">@qq.com</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 收件人邮箱，可添加多个邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;"># 发送邮件的内容（调用模板文件中的）</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">html:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">{{ template &quot;email.html&quot; .}}</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">name:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dingtalk</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">webhook_configs:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">url:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://192.168.88.101:8060/dingtalk/webhook1/send</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">send_resolved:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">true</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 重载alertmanager配置</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/-/reload</span></span></code></pre></div><h3 id="配置触发器-1" tabindex="-1">配置触发器 <a class="header-anchor" href="#配置触发器-1" aria-label="Permalink to &quot;配置触发器&quot;">​</a></h3><p>前置准备已配置，查看现有触发器，访问告警模块的<strong>web</strong>⻚⾯</p><p><a href="http://192.168.11.61:9090/alerts" target="_blank" rel="noreferrer">http://192.168.11.61:9090/alerts</a></p><p>INACTIVE：活跃中，即表示正常⽆告警产⽣。</p><p>PENDING：待触发，表示已经达到预设的阈值，但没达到预设的时间。</p><p>FIRING：表示达到预设的阈值并超过预设的时间触发告警</p><h3 id="测试钉钉" tabindex="-1">测试钉钉 <a class="header-anchor" href="#测试钉钉" aria-label="Permalink to &quot;测试钉钉&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 关闭node-exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># docker环境</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># ⼆进制安装环境</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看prometheus的alerts: http://192.168.11.61:9090/alerts</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看alertmanager的alerts：http://192.168.11.61:9093/#/alerts</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 如果没收到钉钉消息，排查</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">logs</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span></span></code></pre></div><p>钉钉正常收到报警信息如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301537542.png" alt="image-20230430153713389" style="zoom:80%;"><h3 id="可能的问题" tabindex="-1">可能的问题 <a class="header-anchor" href="#可能的问题" aria-label="Permalink to &quot;可能的问题&quot;">​</a></h3><p>由于配置名称错误导致不能发送报警信息给钉钉，如下图：解决方式：修改名称，使其配置正确</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261649896.png" alt="image-20230426164935718" style="zoom:80%;"><h2 id="企业微信告警" tabindex="-1">企业微信告警 <a class="header-anchor" href="#企业微信告警" aria-label="Permalink to &quot;企业微信告警&quot;">​</a></h2><h3 id="注册企业微信" tabindex="-1">注册企业微信 <a class="header-anchor" href="#注册企业微信" aria-label="Permalink to &quot;注册企业微信&quot;">​</a></h3><p>浏览器打开<a href="https://work.weixin.qq.com/" target="_blank" rel="noreferrer">https://work.weixin.qq.com/</a> 点击注册，如下填写资料</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261652311.png" alt="image-20230426165205193" style="zoom:80%;"><h3 id="webhook告警-和微信应用告警二选一-⭐" tabindex="-1">webhook告警（和微信应⽤告警⼆选⼀）⭐ <a class="header-anchor" href="#webhook告警-和微信应用告警二选一-⭐" aria-label="Permalink to &quot;webhook告警（和微信应⽤告警⼆选⼀）⭐&quot;">​</a></h3><p>添加群机器⼈，注册成功后，⼿机下载企业微信，登陆企业微信。在⼿机上，如下图操作：</p><blockquote><p>注：因为我这个是测试企业微信，所以就在”企业全员群“，新建群机器⼈了。真实⼀般都是新创建个部⻔，然后把需要接受报警的⼈拉到这个部⻔⾥⾯，然后在这个部⻔群⾥⾯新建机器⼈。</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261657210.png" alt="image-20230426165709046" style="zoom:80%;"><p>复制机器⼈的<strong>webhook</strong>地址：<a href="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=15957180-9e0c-4224-8c44-08cb09b7f90d" target="_blank" rel="noreferrer">https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=15957180-9e0c-4224-8c44-08cb09b7f90d</a></p><p>我在上图复制的到webhook地址如下：真实只需要⽤到key后⾯的</p><p><strong>docker安装webhook-wechat</strong></p><p>Prometheus服务器上安装</p><p>创建存放webhook-wechat的数据⽬录</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus/webhook-wechat</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-p</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus/webhook-wechat</span></span></code></pre></div><p>使⽤cat创建新⽂件</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker-compose.yaml</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;">&quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">version: &quot;3&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">services:</span></span>
<span class="line"><span style="color:#C3E88D;">  prometheus-webhook-webchat:</span></span>
<span class="line"><span style="color:#C3E88D;">    image: linge365/webhook-wechat:latest</span></span>
<span class="line"><span style="color:#C3E88D;">    container_name: prometheus-webhook-webchat</span></span>
<span class="line"><span style="color:#C3E88D;">    restart: always</span></span>
<span class="line"><span style="color:#C3E88D;">    volumes:</span></span>
<span class="line"><span style="color:#C3E88D;">      - /etc/localtime:/etc/localtime</span></span>
<span class="line"><span style="color:#C3E88D;">    ports:</span></span>
<span class="line"><span style="color:#C3E88D;">      - &quot;5000:5000&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">    environment:</span></span>
<span class="line"><span style="color:#C3E88D;">      ROBOT_TOKEN: &quot;15957180-9e0c-4224-8c44-08cb09b7f90d&quot;</span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker-compose</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">ps</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">logs</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus-webhook-webchat</span></span></code></pre></div><p><strong>修改alertmanager配置</strong></p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager.yml</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#增加如下配置</span></span>
<span class="line"><span style="color:#FFCB6B;">route:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 路由组配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">group_by:</span><span style="color:#A6ACCD;"> [</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">] </span><span style="color:#676E95;font-style:italic;"># 分组依据，可分多个组</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">group_wait:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">10</span><span style="color:#C3E88D;">s</span><span style="color:#A6ACCD;">        </span><span style="color:#676E95;font-style:italic;"># 第一次满足告警条件后等待多久发送邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">group_interval:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">10</span><span style="color:#C3E88D;">s</span><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;"># 出现新告警后等待多久发送邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">repeat_interval:</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">1</span><span style="color:#C3E88D;">h</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 若已经成功发送邮件等待多久后再次发送</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">receiver:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">wechat</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#FFCB6B;">receivers:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 收件组配置</span></span>
<span class="line"><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">name:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">wechat</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">webhook_configs:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">url:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://192.168.88.101:5000</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">send_resolved:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">true</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 重载alertmanager配置</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/-/reload</span></span></code></pre></div><h3 id="微信应用告警-和webhook告警二选一" tabindex="-1">微信应⽤告警（和<strong>webhook</strong>告警⼆选⼀） <a class="header-anchor" href="#微信应用告警-和webhook告警二选一" aria-label="Permalink to &quot;微信应⽤告警（和**webhook**告警⼆选⼀）&quot;">​</a></h3><blockquote><p>必须有域名才能用这种方式，所以目前只能用webhook方式</p></blockquote><p>企业微信应⽤需要添加ip⽩名单才能正常使⽤,浏览器打开企业微信<a href="https://work.weixin.qq.com/" target="_blank" rel="noreferrer">企业微信官⽹</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261703615.png" alt="image-20230426170358443" style="zoom:80%;"><p>⼿机下载“企业微信”，使⽤注册的⼿机登录。扫描⼆维码登录企业微信官⽹</p><p>短信验证</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261704438.png" alt="image-20230426170459298" style="zoom:80%;"><p>创建应⽤</p><p>登录成功后，选择应⽤管理--创建应⽤</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261705612.png" alt="image-20230426170518466" style="zoom:80%;"><p>上传logo，填写应⽤名称，选择可⻅范围</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261705672.png" alt="image-20230426170535541" style="zoom:80%;"><p>获取<strong>AgentID</strong></p><p>创建应⽤成功后，复制AgentId,和查看Secret--会发送Secret到⼿机企业微信中。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261705327.png" alt="image-20230426170553197" style="zoom:80%;"><p>设置<strong>ip</strong>⽩名单</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261706923.png" alt="image-20230426170617796" style="zoom:80%;"><p>设置可信域名</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261706979.png" alt="image-20230426170638855" style="zoom:80%;"><p>获取<strong>Secret</strong></p><p>⼿机下载“企业微信”app，并登录成功。”企业微信团队“发来⼀条新消息，点击查看Secret，复制Secret</p><p>-rg8Xtzchefy6w94O6G_qT5gOMhDZt7MsZmHSELAOZw</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261707154.png" alt="image-20230426170710011" style="zoom:80%;"><p>获取部⻔<strong>id</strong></p><p>注：获取⽤户名或者创建标签都可以</p><p>点击通讯录--选择企业名--点右边--查看部⻔id，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261707227.png" alt="image-20230426170732114" style="zoom:80%;"><p>获取<strong>corp_id</strong></p><p>点击“我的企业”--复制企业id.ww75c7ff0bc812538c</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261707770.png" alt="image-20230426170751651" style="zoom:80%;"><p>修改alertmanager配置</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager/config.yml</span></span>
<span class="line"></span>
<span class="line"><span style="color:#FFCB6B;">route:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">receiver:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">wechat</span></span>
<span class="line"><span style="color:#FFCB6B;">receivers:</span></span>
<span class="line"><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">name:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">wechat</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">wechat_configs:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#FFCB6B;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">send_resolved:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">true</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;">#to_user: &#39;@all&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;">#发送给企业微信⽤户的ID，@all是所有⼈</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#676E95;font-style:italic;">#to_tag: &#39;1&#39; # 企业微信中创建的接收告警的标签</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">to_party:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">1</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 部⻔id</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">agent_id:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">1000002</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;">#企业微信中创建的应⽤的ID</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">corp_id:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">ww75c7ff0bc812538c</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;">#企业微信中企业ID</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#FFCB6B;">api_secret:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">-rg8Xtzchefy6w94O6G_qT5gOMhDZt7MsZmHSELAOZw</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 企业微信中，应⽤的Secret</span></span></code></pre></div><p>检查配置</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># Docker安装⽅式，检查</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">exec</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-it</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">amtool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">check-config</span></span>
<span class="line"><span style="color:#FFCB6B;">/etc/alertmanager/config.yml</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># ⼆进制安装⽅式，检查</span></span>
<span class="line"><span style="color:#FFCB6B;">/opt/alertmanager/alertmanager</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">amtool</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">check-config</span></span>
<span class="line"><span style="color:#FFCB6B;">/etc/alertmanager/config.yml</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 重载alertmanager配置</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/-/reload</span></span></code></pre></div><h3 id="测试微信告警" tabindex="-1">测试微信告警 <a class="header-anchor" href="#测试微信告警" aria-label="Permalink to &quot;测试微信告警&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 关闭node-exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># Docker环境</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># ⼆进制安装环境</span></span>
<span class="line"><span style="color:#FFCB6B;">systemctl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看prometheus的alerts: http://192.168.11.61:9090/alerts</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 查看alertmanager的alerts：http://192.168.11.61:9093/#/alerts如果没收到钉钉消息，排查</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">logs</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-f</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301555843.png" alt="image-20230430155557690" style="zoom:50%;"><p>微信正常收到报警信息如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304261712343.png" alt="image-20230426171203189" style="zoom:80%;"><h3 id="使用模版-非必需-仅限微信应用告警" tabindex="-1">使⽤模版(⾮必需，仅限微信应⽤告警) <a class="header-anchor" href="#使用模版-非必需-仅限微信应用告警" aria-label="Permalink to &quot;使⽤模版(⾮必需，仅限微信应⽤告警)&quot;">​</a></h3><blockquote><p>看需求--不使⽤模版默认也⾏，<strong>不适⽤webhook⽅式告警。</strong></p></blockquote><p><strong>1</strong>、创建模版⽂件（<strong>prometheus</strong>服务器操作）</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/module/alertmanager-0.23.0</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 创建存放模版的⽬录</span></span>
<span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">template</span></span></code></pre></div><p>通过cat创建</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">template/wechat.tmpl</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;">&quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot; </span></span>
<span class="line"><span style="color:#C3E88D;">{{ define &quot;wechat.html&quot; }}</span></span>
<span class="line"><span style="color:#C3E88D;">{{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }}</span></span>
<span class="line"><span style="color:#C3E88D;">@告警通知</span></span>
<span class="line"><span style="color:#C3E88D;">告警程序: prometheus_alert</span></span>
<span class="line"><span style="color:#C3E88D;">告警级别: {{ .Labels.severity }}级别</span></span>
<span class="line"><span style="color:#C3E88D;">告警类型: {{ .Labels.alertname }}</span></span>
<span class="line"><span style="color:#C3E88D;">故障主机: {{ .Labels.instance }}</span></span>
<span class="line"><span style="color:#C3E88D;">告警主题: {{ .Annotations.summary }}</span></span>
<span class="line"><span style="color:#C3E88D;">告警详情: {{ .Annotations.description }}</span></span>
<span class="line"><span style="color:#C3E88D;">触发时间: {{ .StartsAt.Local.Format &quot;2006-01-02 15:04:05&quot; }}</span></span>
<span class="line"><span style="color:#C3E88D;">{{ end }}{{ end -}}</span></span>
<span class="line"><span style="color:#C3E88D;">{{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }}</span></span>
<span class="line"><span style="color:#C3E88D;">@告警恢复</span></span>
<span class="line"><span style="color:#C3E88D;">告警程序: prometheus_alert</span></span>
<span class="line"><span style="color:#C3E88D;">故障主机: {{ .Labels.instance }}</span></span>
<span class="line"><span style="color:#C3E88D;">故障主题: {{ .Annotations.summary }}</span></span>
<span class="line"><span style="color:#C3E88D;">告警详情: {{ .Annotations.description }}</span></span>
<span class="line"><span style="color:#C3E88D;">告警时间: {{ .StartsAt.Local.Format &quot;2006-01-02 15:04:05&quot; }}</span></span>
<span class="line"><span style="color:#C3E88D;">恢复时间: {{ .EndsAt.Local.Format &quot;2006-01-02 15:04:05&quot; }}</span></span>
<span class="line"><span style="color:#C3E88D;">{{ end }}{{ end -}}</span></span>
<span class="line"><span style="color:#C3E88D;">{{- end }}</span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><p>增加message这⾏</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;">#模版配置</span></span>
<span class="line"><span style="color:#F07178;">templates</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">/opt/module/alertmanager-0.23.0/template/*.tmpl</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#FFCB6B;">...</span><span style="color:#F78C6C;">.</span></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">wechat</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">wechat_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span>
<span class="line"><span style="color:#89DDFF;">    </span><span style="color:#676E95;font-style:italic;"># 只增加这⾏配置</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">message</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">{{ template &quot;wechat.html&quot; . }}</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/-/reload</span></span></code></pre></div><p>检查：<a href="http://192.168.11.61:9093/#/status" target="_blank" rel="noreferrer">http://192.168.11.61:9093/#/status</a></p><h2 id="抑制和静默" tabindex="-1">抑制和静默 <a class="header-anchor" href="#抑制和静默" aria-label="Permalink to &quot;抑制和静默&quot;">​</a></h2><h3 id="抑制机制" tabindex="-1">抑制机制 <a class="header-anchor" href="#抑制机制" aria-label="Permalink to &quot;抑制机制&quot;">​</a></h3><blockquote><p>Alertmanager的抑制机制可以避免当某种问题告警产⽣之后⽤户接收到⼤量由此问题导致的⼀系列的其它告警通知。例如当集群不可⽤时，⽤户可能只希望接收到⼀条告警，告诉他这时候集群出现了问题，⽽不是⼤量的如集群中的应⽤异常、中间件服务异常的告警通知。</p></blockquote><blockquote><p>在Alertmanager配置⽂件中，使⽤inhibit_rules定义⼀组告警的抑制规则</p></blockquote><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">inhibit_rules:</span></span>
<span class="line"><span style="color:#A6ACCD;">  [ - &lt;inhibit_rule&gt; ... ]</span></span></code></pre></div><p>每⼀条抑制规则的具体配置如下：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">target_match:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelname</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">: </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelvalue</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">, ... </span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#FFCB6B;">target_match_re:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelname</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">: </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">regex</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">, ... </span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#FFCB6B;">source_match:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelname</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">: </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelvalue</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">, ... </span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#FFCB6B;">source_match_re:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelname</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">: </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">regex</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">, ... </span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> equal: </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&lt;</span><span style="color:#A6ACCD;">labelname</span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;">, ... </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">]</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">]</span></span></code></pre></div><blockquote><p>当已经发送的告警通知匹配到target_match和target_match_re规则，当有新的告警规则如果满⾜source_match或者定义的匹配规则，并且已发送的告警与新产⽣的告警中equal定义的标签完全相同，则启动抑制机制，新告警不会发送</p></blockquote><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">inhibit_rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">critical</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">target_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">warning</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">equal</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dev</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">instance</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><p>在alertname、dev、instance 三个标签的值相同情况下，critaical 的报警会抑制 warning 级别的报警信息。</p><h3 id="临时静默" tabindex="-1">临时静默 <a class="header-anchor" href="#临时静默" aria-label="Permalink to &quot;临时静默&quot;">​</a></h3><blockquote><p>静默提供了⼀个简单的机制可以快速根据标签对告警进⾏静默处理。如果接收到的告警符合静默的配置，</p><p>Alertmanager则不会发送告警通知。静默设置需要在Alertmanager的Werb⻚⾯上进⾏设置。</p></blockquote><blockquote><p>除了基于抑制机制可以控制告警通知的⾏为以外，⽤户或者管理员还可以直接通过Alertmanager的UI临时屏蔽特定的告警通知。通过定义标签的匹配规则(字符串或者正则表达式)，如果新的告警通知满⾜静默规则的设置，则停⽌向receiver发送通知。⽤于停机维护，或者有⼀个不需要处理的告警信息</p></blockquote><h4 id="创建静默规则" tabindex="-1">创建静默规则 <a class="header-anchor" href="#创建静默规则" aria-label="Permalink to &quot;创建静默规则&quot;">​</a></h4><p>进⼊Alertmanager UI，点击&quot;Silences&quot;---在点右上⻆的&quot;New Silence&quot;显示如下内容：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270953848.png" alt="image-20230427095309671" style="zoom:80%;"><blockquote><p>定义静默规则的开始时间以及持续时间和结束时间，通过Matchers部分可以设置多条匹配规则(字符串匹配或者正则匹配)。填写当前静默规则的创建者以及创建原因后，点击&quot;Create&quot;按钮即可。通过&quot;Preview Alerts&quot;可以查看预览当前匹配规则匹配到的告警信息。静默规则创建成功后，Alertmanager会开始加载该规则并且设置状态为Pending,当规则⽣效后则进⾏到Active状态。</p></blockquote><h4 id="查看静默规则" tabindex="-1">查看静默规则 <a class="header-anchor" href="#查看静默规则" aria-label="Permalink to &quot;查看静默规则&quot;">​</a></h4><p>点击”silences“在“Active”查看正在运⾏的静默规则</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270953557.png" alt="image-20230427095346397" style="zoom:80%;"><h4 id="匹配的告警信息失效" tabindex="-1">匹配的告警信息失效 <a class="header-anchor" href="#匹配的告警信息失效" aria-label="Permalink to &quot;匹配的告警信息失效&quot;">​</a></h4><p>当静默规则⽣效以后，从Alertmanager的Alerts⻚⾯下⽤户将不会看到该规则匹配到的告警信息。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304270954848.png" alt="image-20230427095406696" style="zoom:80%;"><h4 id="取消静默规则" tabindex="-1">取消静默规则 <a class="header-anchor" href="#取消静默规则" aria-label="Permalink to &quot;取消静默规则&quot;">​</a></h4><p>对于已经⽣效的规则，⽤户可以通过⼿动点击”Expire“按钮使当前规则过期。</p><h3 id="路由匹配" tabindex="-1">路由匹配 <a class="header-anchor" href="#路由匹配" aria-label="Permalink to &quot;路由匹配&quot;">​</a></h3><blockquote><p>每⼀个告警都会从配置⽂件中顶级的route进⼊路由树，需要注意的是顶级的route必须匹配所有告警(即不能有任何的匹配设置match和match_re)，每⼀个路由都可以定义⾃⼰的接受⼈以及匹配规则。默认情况下，告警进⼊到顶级route后会遍历所有的⼦节点，直到找到最深的匹配route，并将告警发送到该route定义的receiver中。但如果route中设置<strong>continue</strong>的值为false，那么告警在匹配到第⼀个⼦节点之后就直接停⽌。如果<strong>continue</strong>为true，报警则会继续进⾏后续⼦节点的匹配。如果当前告警匹配不到任何的⼦节点，那该告警将会基于当前路由节点的接收器配置⽅式进⾏处理。</p></blockquote><blockquote><p>其中告警的匹配有两种⽅式可以选择。⼀种⽅式基于字符串验证，通过设置<strong>match</strong>规则判断当前告警中是否存在标签labelname并且其值等于labelvalue。第⼆种⽅式则基于正则表达式，通过设置<strong>match_re</strong>验证当前告警标签的值是否满⾜正则表达式的内容。如果警报已经成功发送通知, 如果想设置发送告警通知之前要等待时间，则可以通过<strong>repeat_interval</strong>参数进⾏设置。</p></blockquote><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 定义分组,根据label标签进⾏分组</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 分组等待时间，也就是说在10秒内同⼀个组中有没有⼀起报警的，如果有则同时发出报警邮件，也就是有2个报警同时发在⼀个邮件</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 告警时间间隔</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">repeat_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10m</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 重复告警间隔，也就是触发的⼀个告警在10分钟内没有处理则再次发⼀封邮件。</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">continue</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">false</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 若路由上的continue字段的值为false，则遇到第⼀个匹配的路由分⽀后即终⽌。否则，将继续匹配后续的⼦节点;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-01</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 默认邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">routes</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 启⽤⼀个⼦路由</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-dba</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">  </span><span style="color:#676E95;font-style:italic;"># 接收者为receiver-dba</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 分组等待时间</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match_re</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;">  </span><span style="color:#676E95;font-style:italic;"># 匹配⼀个正则</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">service</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mysql|db</span><span style="color:#A6ACCD;">  </span><span style="color:#676E95;font-style:italic;"># service标签包含mysql和db的统⼀发送给dba的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">continue</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">false</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 若路由上的continue字段的值为false，则遇到第⼀个匹配的路由分⽀后即终⽌。否则，将继续匹配后续的⼦节点;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-01</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">  </span><span style="color:#676E95;font-style:italic;"># 接收者为receiver-01</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 分组时间</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">serverity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">error</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 将serverity标签值包含error的发送给yunwei的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">continue</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">false</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 若路由上的continue字段的值为false，则遇到第⼀个匹配的路由分⽀后即终⽌。否则，将继续匹配后续的⼦节点;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 定义接收者的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-01</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 接收者名字，要和routes中的receiver对应</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">11111@qq.com</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># receiver-01的邮箱地址</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-dba</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">  </span><span style="color:#676E95;font-style:italic;"># 接收者名字，要和routes中的receiver对应</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">2222@qq.com</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># receiver-dba的邮箱地址</span></span></code></pre></div><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">instance</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 根据 instance 标签分组</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">continue</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 为true则还需要去匹配⼦路由。</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-01</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">routes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-01</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">alertname</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">InstanceDown</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 告警的名字是 InstanceDown 则发送给receiver-03</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">webchat</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match_re</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">alertname</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">Cpu.*</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;">  </span><span style="color:#676E95;font-style:italic;"># 告警的名字以 Cpu开头的 则发送给 webchat</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dingtalk</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">alertname</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">InstanceDown</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 告警的名字是 InstanceDown 则发送给 dingtalk</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">receiver-01</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">1111@qq.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">webchat</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">webhook_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">url</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://192.168.11.61:5000</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dingtalk</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">webhook_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">url</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://192.168.11.61:8060/dingtalk/webhook1/send</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span></code></pre></div><h3 id="告警分组" tabindex="-1">告警分组 <a class="header-anchor" href="#告警分组" aria-label="Permalink to &quot;告警分组&quot;">​</a></h3><blockquote><p>在之前的部分有讲过，Alertmanager可以对告警通知进⾏分组，将多条告警合合并为⼀个通知。这⾥我们可以使⽤<strong>group_by</strong>来定义分组规则。基于告警中包含的标签，如果满⾜<strong>group_by</strong>中定义标签名称，那么这些告警将会合并为⼀个通知发送给接收器。</p></blockquote><blockquote><p>有的时候为了能够⼀次性收集和发送更多的相关信息时，可以通过<strong>group_wait</strong>参数设置等待时间，如果在等待时间内当前group接收到了新的告警，这些告警将会合并为⼀个通知向receiver发送。⽽<strong>group_interval</strong>配置，则⽤于定义相同的Group之间发送告警通知的时间间隔。例如，当使⽤Prometheus监控多个集群以及部署在集群中的应⽤和数据库服务，并且定义以下的告警处理路由规则来对集群中的异常进⾏通知。</p></blockquote><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">default-receiver</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">30s</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">5m</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">repeat_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">4h</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">cluster</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#F07178;">routes</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">database-pager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">match_re</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">service</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mysql|cassandra</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">frontend-pager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">product</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">environment</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">team</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">frontend</span></span></code></pre></div><h2 id="告警总结⭐" tabindex="-1">告警总结⭐ <a class="header-anchor" href="#告警总结⭐" aria-label="Permalink to &quot;告警总结⭐&quot;">​</a></h2><h3 id="生产建议" tabindex="-1">⽣产建议 <a class="header-anchor" href="#生产建议" aria-label="Permalink to &quot;⽣产建议&quot;">​</a></h3><blockquote><p>最好选2个告警通知。如果很重要的可以设置电话告警，因为微信，钉钉信息很多⼈不看。但是来电话了基本都看</p></blockquote><h3 id="配置总结" tabindex="-1">配置总结 <a class="header-anchor" href="#配置总结" aria-label="Permalink to &quot;配置总结&quot;">​</a></h3><h3 id="完整配置" tabindex="-1">完整配置 <a class="header-anchor" href="#完整配置" aria-label="Permalink to &quot;完整配置&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">alertmanager.yml</span></span></code></pre></div><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#163服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_smarthost</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">smtp.163.com:465</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#发邮件的邮箱</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_from</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#发邮件的邮箱⽤户名，也就是你的邮箱　　　　　</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_auth_username</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#发邮件的邮箱密码</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_auth_password</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">MONBUFHOQZPBXHGD</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;">#进⾏tls验证</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">smtp_require_tls</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">false</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">templates</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">/etc/alertmanager/template/*.tmpl</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#F07178;">route</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">group_by</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#89DDFF;"> </span><span style="color:#676E95;font-style:italic;"># 当收到告警的时候，等待group_wait配置的时间，看是否还有告警，如果有就⼀起发出去</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">group_wait</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#89DDFF;"> </span><span style="color:#676E95;font-style:italic;"># 如果上次告警信息发送成功，此时⼜来了⼀个新的告警数据，则需要等待group_interval配置的时间才可以发送出去</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">group_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#89DDFF;"> </span><span style="color:#676E95;font-style:italic;"># 如果上次告警信息发送成功，且问题没有解决，则等待 repeat_interval配置的时间再次发送告警数据</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">repeat_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10m</span></span>
<span class="line"><span style="color:#89DDFF;"> </span><span style="color:#676E95;font-style:italic;"># 全局报警组，这个参数是必选的</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">receiver</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">email_dingtalk</span></span>
<span class="line"></span>
<span class="line"><span style="color:#F07178;">receivers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">email_dingtalk</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">email_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">to</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cdring@163.com</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">html</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">{{ template &quot;email.html&quot; .}}</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">webhook_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">url</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://192.168.11.61:8060/dingtalk/webhook1/send</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">wechat</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">webhook_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">url</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://192.168.11.61:5000</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">ca</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">webhook_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">url</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">http://api.aiops.com/alert/api/event/prometheus/cd0ffd35ec9a4ac88dc944dc470d603b</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">send_resolved</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#FF9CAC;">true</span></span>
<span class="line"><span style="color:#F07178;">inhibit_rules</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">critical</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">target_match</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">severity</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">warning</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">equal</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertname</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">dev</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">instance</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><h3 id="配置告警规则⭐" tabindex="-1">配置告警规则⭐ <a class="header-anchor" href="#配置告警规则⭐" aria-label="Permalink to &quot;配置告警规则⭐&quot;">​</a></h3><p>规则：<a href="https://github.com/samber/awesome-prometheus-alerts" target="_blank" rel="noreferrer">https://github.com/samber/awesome-prometheus-alerts</a></p><p>该地址包含了大多数的告警规则，可以直接去复制就行</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301810104.png" alt="image-20230430181023946" style="zoom:80%;"><p><a href="https://samber.github.io/awesome-prometheus-alerts/rules#mysql" target="_blank" rel="noreferrer">https://samber.github.io/awesome-prometheus-alerts/rules#mysql</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301811525.png" alt="image-20230430181106320" style="zoom:80%;"><h1 id="服务发现" tabindex="-1">服务发现 <a class="header-anchor" href="#服务发现" aria-label="Permalink to &quot;服务发现&quot;">​</a></h1><h2 id="服务发现-1" tabindex="-1">服务发现 <a class="header-anchor" href="#服务发现-1" aria-label="Permalink to &quot;服务发现&quot;">​</a></h2><blockquote><p>接下来我们将学习 Prometheus 中是如何使⽤服务发现来查找和抓取⽬标的。我们知道在 Prometheus 配置⽂件中可以通过⼀个 static_configs 来配置静态的抓取任务，但是在云环境下，特别是容器环境下，抓取⽬标地址是经常变动的，所以⽤静态的⽅式就不能满⾜这些场景了，还有特别在很多服务器需要监控时。所以我们需要监控系统能够动态感知这个变化，不可能每次变动都去⼿动重新配置的，为了应对复杂的动态环境，Prometheus 也提供了与基础设施中的服务发现集成的功能。</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271052768.png" alt="image-20230427105222560" style="zoom:67%;"><p>Prometheus 已经⽀持多种内置的服务发现机制：</p><blockquote><p>发现云服务商的 VM 虚拟机</p><p>Kubernetes 上的⾃动发现</p><p>通⽤的服务查找，例如 DNS、Consul、Zookeeper 或⾃定义发现机制</p></blockquote><p>我们都可以通过 Prometheus 配置⽂件中的 scrape_config 部分进⾏配置，Prometheus 会不断更新动态的抓取⽬标列表，⾃动停⽌抓取旧的实例，开始抓取新的实例，Prometheus 特别适合运⾏于Kubernetes 集群下⾯，可以⾃动发现监控⽬标。此外⼤部分服务发现机制还会提供⽬标的⼀些元数据，通常都是带有 __ 的前缀， ⽐如标签、注解、服务名等等，可以在 relabeling 阶段使⽤这些元数据来过滤修改⽬标，这些元信息标签在重新标记阶段后被删除。</p><h2 id="基于文件的服务发现⭐" tabindex="-1">基于文件的服务发现⭐ <a class="header-anchor" href="#基于文件的服务发现⭐" aria-label="Permalink to &quot;基于文件的服务发现⭐&quot;">​</a></h2><p>除了基于 Consul 的服务发现之外，Prometheus 也允许我们进⾏⾃定义的发现集成，可以通过 watch ⼀组本地⽂件来获取抓取⽬标以及标签信息，也就是我们常说的基于⽂件的服务发现⽅式。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271053172.png" alt="image-20230427105327949" style="zoom:67%;"><blockquote><p>基于⽂件的服务发现提供了⼀种更通⽤的⽅式来配置静态⽬标，并作为⼀个接⼝插⼊⾃定义服务发现机制。它读取⼀组包含零个或多个 &lt;static_config&gt; 列表的⽂件，对所有定义的⽂件的变更通过磁盘监视被检测到并⽴即应⽤，⽂件可以以 YAML 或 JSON 格式提供。⽂件必须包含⼀个静态配置的列表:当然该⽂件也可以使⽤ JSON 格式进⾏配置：</p></blockquote><div class="language-json"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">[</span></span>
<span class="line"><span style="color:#A6ACCD;">   </span><span style="color:#89DDFF;">{</span></span>
<span class="line"><span style="color:#A6ACCD;">     </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C792EA;">targets</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">&lt;host&gt;</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> ... </span><span style="color:#89DDFF;">],</span></span>
<span class="line"><span style="color:#A6ACCD;">     </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C792EA;">labels</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">{</span></span>
<span class="line"><span style="color:#A6ACCD;">       </span><span style="color:#89DDFF;">&quot;</span><span style="color:#FFCB6B;">&lt;labelname&gt;</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">&lt;labelvalue&gt;</span><span style="color:#89DDFF;">&quot;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> ...</span></span>
<span class="line"><span style="color:#A6ACCD;">     </span><span style="color:#89DDFF;">}</span></span>
<span class="line"><span style="color:#A6ACCD;">   </span><span style="color:#89DDFF;">},</span></span>
<span class="line"><span style="color:#A6ACCD;">   ...</span></span>
<span class="line"><span style="color:#89DDFF;">]</span></span></code></pre></div><p>如果是 YAML ⽂件则格式为：</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> - </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">&lt;host&gt;</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">&lt;labelname&gt;</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;labelvalue&gt; ...</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">]</span></span></code></pre></div><p>⽂件内容也会在指定的刷新间隔时间内定期重新读取。</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># Patterns for files from which target groups are extracted.</span></span>
<span class="line"><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> - </span><span style="color:#C3E88D;">&lt;filename_pattern&gt; ...</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># Refresh interval to re-read the files.</span></span>
<span class="line"><span style="color:#89DDFF;">[</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">&lt;duration&gt; | default = 5m</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">]</span></span></code></pre></div><p>其中 &lt;filename<em>pattern&gt; 可以是⼀个以 .json 、 .yml 或 .yaml 结尾的路径，最后⼀个路径段可以包含⼀个匹配任何字符序列的 * ，例如： my/path/tg_</em>.json 。</p><h3 id="创建文件" tabindex="-1">创建⽂件 <a class="header-anchor" href="#创建文件" aria-label="Permalink to &quot;创建⽂件&quot;">​</a></h3><p>接下来我们来创建⼀个⽤于服务发现的⽬标⽂件，在与 prometheus.yml ⽂件相同⽬录下⾯创建⼀个名为 targets.yml 的⽂件，内容如下所示：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/module/prometheus-2.29.1</span></span>
<span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets</span></span></code></pre></div><p>targets.yml</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">localhost:9090</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">cadvisor:8080</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">cadvisor</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8080</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">cadvisor</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">node_exporter:9100</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">Prometheus服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9100</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9113</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">nginx_exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9121</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">redis_exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9419</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">rabitmq_exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9216</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mongodb_exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9104</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">mysqld_exporter</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9256</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">instance</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">test服务器</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">job</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">process</span></span></code></pre></div><p>springboot.yml</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat &gt;springboot.yml&lt;&lt;&quot;EOF&quot;</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:8081</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><p>blackbox-exporter-http.yml</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat &gt;blackbox-exporter-http.yml&lt;&lt;&quot;EOF&quot;</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://www.baidu.com</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://www.jd.com</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><p>blackbox-exporter-tcp.yml</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat &gt;blackbox-exporter-tcp.yml&lt;&lt;&quot;EOF&quot;</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:22</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9090</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><p>blackbox-exporter-icmp.yml</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat &gt;blackbox-exporter-icmp.yml&lt;&lt;&quot;EOF&quot;</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">192.168.88.101</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">192.168.88.101</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><p>domain.yml</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#C3E88D;">cat &gt;domain.yml&lt;&lt;&quot;EOF&quot;</span></span>
<span class="line"><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">qq.com</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">baidu.com</span></span>
<span class="line"><span style="color:#C3E88D;">EOF</span></span></code></pre></div><h3 id="配置文件服务发现" tabindex="-1">配置⽂件服务发现 <a class="header-anchor" href="#配置文件服务发现" aria-label="Permalink to &quot;配置⽂件服务发现&quot;">​</a></h3><p>⽤于发现的⽬标⽂件创建完成后，要让 Prometheus 能够从上⾯的 targets.yml ⽂件中⾃动读取抓取⽬标，需要在 prometheus.yml 配置⽂件中的 scrape_configs 部分添加如下所示的抓取配置：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 备份⽂件，变成prometheus.yml.bak</span></span>
<span class="line"><span style="color:#FFCB6B;">cp</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-a</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.yml{,.bak}</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 修改prometheus.yml</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.yml</span></span></code></pre></div><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 全局配置</span></span>
<span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">evaluation_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># Alertmanager 配置</span></span>
<span class="line"><span style="color:#F07178;">alerting</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">alertmanagers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 报警(触发器)配置</span></span>
<span class="line"><span style="color:#F07178;">rule_files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">alert.yml</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">rules/*.yml</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 搜刮配置</span></span>
<span class="line"><span style="color:#F07178;">scrape_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">file-sd-test</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">file_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">targets/targets.yml</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># Spring Boot 2.x 应⽤数据采集配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">file-springboot-demo</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">/actuator/prometheus</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">file_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets/springboot.yml</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># HTTP 配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">file-blackbox_http</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">params</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">module</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">http_2xx</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">file_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets/blackbox-exporter-http.yml</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__address__</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__param_target</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9115</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># TCP 检查配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">file-blackbox_tcp</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">params</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">module</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">tcp_connect</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">file_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets/blackbox-exporter-tcp.yml</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__address__</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__param_target</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9115</span></span>
<span class="line"></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># ICMP 检查配置 (Ping)</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">blackbox_icmp</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">params</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">module</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">icmp</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">file_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets/blackbox-exporter-http.yml</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__address__</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__param_target</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9115</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">domain</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__address__</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9222</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># domain_exporter address</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">file_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">refresh_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">          </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">targets/domain.yml</span></span></code></pre></div><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">curl -X POST :9090/-/reload</span></span></code></pre></div><p>重新 reload 或者重启下 Prometheus 让其重新读取配置⽂件信息，然后同样前往 Prometheus UI 的targets ⻚⾯下⾯查看</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271102658.png" alt="image-20230427110206365" style="zoom:80%;"><blockquote><p>然后我们可以尝试改变 targets.yml 的内容，⽐如为 192.168.11.62:8080 实例增加⼀个 env: test的标签，不⽤重新加载 Prometheus 配置，Prometheus 将 watch 该⽂件，并⾃动接收任何变化。</p></blockquote><blockquote><p>注意：当在⽣产环境 Prometheus 服务器中改变 file_sd ⽬标⽂件时，确保改变是原⼦的，以避免重新加载出现错误，最好的⽅法是在⼀个单独的位置创建更新的⽂件，然后将其重命名为⽬标⽂件名（使⽤ mv 命令或 rename() 系统调⽤）。</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304271102933.png" alt="image-20230427110230671" style="zoom:80%;"><blockquote><p>这样我们就完成了基于⽂件的通⽤服务发现机制，<strong>可以让我们动态地改变 Prometheus 的监控⽬标，⽽不需要重新启动或重新加载 Prometheus 服务</strong>。</p></blockquote><h2 id="基于-consul-的服务发现" tabindex="-1">基于 Consul 的服务发现 <a class="header-anchor" href="#基于-consul-的服务发现" aria-label="Permalink to &quot;基于 Consul 的服务发现&quot;">​</a></h2><blockquote><p>Consul 是由 HashiCorp 开发的⼀个⽀持多数据中⼼的分布式服务发现和键值对存储服务的开源软件，是⼀个通⽤的服务发现和注册中⼼⼯具，被⼤量应⽤于基于微服务的软件架构当中。我们通过api将exporter服务注册到 Consul，然后配置 Prometheus 从 Consul 中发现实例。</p></blockquote><blockquote><p>关于 Consul本身的使⽤可以查看官⽅⽂档 <a href="https://learn.hashicorp.com/consul" target="_blank" rel="noreferrer">https://learn.hashicorp.com/consul</a> 了解更多。</p></blockquote><h3 id="二进制安装配置-二选一" tabindex="-1">二进制安装配置（二选⼀） <a class="header-anchor" href="#二进制安装配置-二选一" aria-label="Permalink to &quot;二进制安装配置（二选⼀）&quot;">​</a></h3><p>在⻚⾯ <a href="https://www.consul.io/downloads" target="_blank" rel="noreferrer">https://www.consul.io/downloads</a> 下载符合⾃⼰系统的安装⽂件，⽐如我们这⾥是 Linux 系统，</p><p>使⽤下⾯命令下载安装即可：</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">wget</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">https://releases.hashicorp.com/consul/1.14.5/consul_1.14.5_linux_amd64.zip</span></span>
<span class="line"><span style="color:#FFCB6B;">apt</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">install</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">unzip</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-y</span></span>
<span class="line"><span style="color:#FFCB6B;">unzip</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">consul_1.14.5_linux_amd64.zip</span></span>
<span class="line"><span style="color:#FFCB6B;">mv</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">consul</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/usr/local/bin</span></span>
<span class="line"><span style="color:#FFCB6B;">consul</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">version</span></span></code></pre></div><p>启动<strong>consul</strong>：为了查看更多的⽇志信息，我们可以在 dev 模式下运⾏ Consul，如下所示：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">consul agent -dev -client 0.0.0.0</span></span></code></pre></div><p>启动命令后⾯使⽤ -client 参数指定了客户端绑定的 IP 地址，默认为 127.0.0.1</p><h3 id="docker安装consul-二选一-⭐" tabindex="-1">Docker安装Consul（二选⼀）⭐ <a class="header-anchor" href="#docker安装consul-二选一-⭐" aria-label="Permalink to &quot;Docker安装Consul（二选⼀）⭐&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">run</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--name</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">consul</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-p</span><span style="color:#A6ACCD;"> </span><span style="color:#F78C6C;">8500</span><span style="color:#C3E88D;">:8500</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">consul:1.14.5</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">ps</span></span></code></pre></div><h3 id="访问测试⭐" tabindex="-1">访问测试⭐ <a class="header-anchor" href="#访问测试⭐" aria-label="Permalink to &quot;访问测试⭐&quot;">​</a></h3><p><a href="http://192.168.88.101:8500/ui/dc1/services" target="_blank" rel="noreferrer">http://192.168.88.101:8500/ui/dc1/services</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304302024308.png" alt="image-20230430202454102" style="zoom:80%;"><h3 id="服务注册⭐" tabindex="-1">服务注册⭐ <a class="header-anchor" href="#服务注册⭐" aria-label="Permalink to &quot;服务注册⭐&quot;">​</a></h3><h4 id="使用命令行注册" tabindex="-1">使⽤命令⾏注册 <a class="header-anchor" href="#使用命令行注册" aria-label="Permalink to &quot;使⽤命令⾏注册&quot;">​</a></h4><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">PUT</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">{&quot;id&quot;: &quot;node1&quot;,&quot;name&quot;: &quot;node_exporter&quot;,&quot;address&quot;:&quot;node_exporter&quot;,&quot;port&quot;: 9100,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;:&quot;node_exporter&quot;,&quot;instance&quot;: &quot;Prometheus服务器&quot;},&quot;checks&quot;: [{&quot;http&quot;:&quot;http://192.168.88.101:9100/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}</span><span style="color:#89DDFF;">&#39;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:8500/v1/agent/service/register</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304302030901.png" alt="image-20230430203051689" style="zoom:80%;"><p>把json数据放在⽂件中，使⽤这个json⽂件注册</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">mkdir</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/consul</span></span>
<span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/consul</span></span>
<span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node_exporter.json</span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;">&quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">   {</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;id&quot;: &quot;node2&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;name&quot;: &quot;node_exporter&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;address&quot;: &quot;192.168.88.101&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;port&quot;: 9100,</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;tags&quot;: [&quot;exporter&quot;],</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;meta&quot;: {</span></span>
<span class="line"><span style="color:#C3E88D;">        &quot;job&quot;: &quot;node_exporter&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">        &quot;instance&quot;: &quot;test服务器&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">   },</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;checks&quot;: [{</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;http&quot;: &quot;http://192.168.88.101:9100/metrics&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">     &quot;interval&quot;: &quot;10s&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">   }]</span></span>
<span class="line"><span style="color:#C3E88D;"> } </span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><h4 id="使用json文件注册" tabindex="-1">使⽤json⽂件注册 <a class="header-anchor" href="#使用json文件注册" aria-label="Permalink to &quot;使⽤json⽂件注册&quot;">​</a></h4><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--request</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">PUT</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--data</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">@node_exporter.json</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:8500/v1/agent/service/register</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304302031150.png" alt="image-20230430203152947" style="zoom:80%;"><p>除了我们注册的 2 个 demo 服务之外，Consul agent 还会将⾃⼰注册为⼀个名为 consul 的服务，我们可以在浏览器中访问 <a href="http://192.168.11.61:8500" target="_blank" rel="noreferrer">http://192.168.11.61:8500</a> 查看注册的服务</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301837116.png" alt="image-20230430183708973" style="zoom:80%;"><p>在 Consul UI ⻚⾯中可以看到有 consul 和 node_exporter 两个 Service 服务。</p><h3 id="配置-prometheus" tabindex="-1">配置 <strong>Prometheus</strong> <a class="header-anchor" href="#配置-prometheus" aria-label="Permalink to &quot;配置 **Prometheus**&quot;">​</a></h3><p>上⾯我们通过 Consul 注册了 2 个 node_exporter 服务，接下来我们将配置 Prometheus 通过 Consul 来⾃动发现 node_porter服务。在 Prometheus 的配置⽂件 prometheus.yml ⽂件中的 scrape_configs 部分添加如下所示的抓取配置：</p><h4 id="备份源文件" tabindex="-1">备份源⽂件 <a class="header-anchor" href="#备份源文件" aria-label="Permalink to &quot;备份源⽂件&quot;">​</a></h4><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/opt/module/prometheus-2.29.1</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 备份⽂件，变成prometheus.yml.bak</span></span>
<span class="line"><span style="color:#FFCB6B;">cp</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-a</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.yml{,.bak}</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 修改prometheus.yml</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus.yml</span></span></code></pre></div><p>使⽤cat去掉之前的配置,使⽤下⾯的配置</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 全局配置</span></span>
<span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;">     </span><span style="color:#C3E88D;">15s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 将搜刮间隔设置为每15秒一次。默认是每1分钟一次。</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">evaluation_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;"># 每15秒评估一次规则。默认是每1分钟一次。</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># Alertmanager 配置</span></span>
<span class="line"><span style="color:#F07178;">alerting</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">alertmanagers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 报警(触发器)配置</span></span>
<span class="line"><span style="color:#F07178;">rule_files</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">alert.yml</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">rules/*.yml</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 搜刮配置</span></span>
<span class="line"><span style="color:#F07178;">scrape_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">prometheus</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">    </span><span style="color:#676E95;font-style:italic;"># 覆盖全局默认值，每15秒从该作业中刮取一次目标</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">localhost:9090</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#89DDFF;">    </span><span style="color:#676E95;font-style:italic;"># 覆盖全局默认值，每15秒从该作业中刮取一次目标</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">15s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">alertmanager:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">consul_exporter</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">consul_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">server</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8500</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*exporter.*</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#  Spring Boot 2.x 应用数据采集配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">consul_springboot_demo</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">/actuator/prometheus</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">5s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">consul_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">server</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8500</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*springboot.*</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#http配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">consul-blackbox_http</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">params</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">module</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">http_2xx</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">consul_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">server</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8500</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*blackbox_http.*</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_service_address</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__param_target</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9115</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#tcp检查配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">consul_blackbox_tcp</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">params</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">module</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">tcp_connect</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">consul_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">server</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8500</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*blackbox_tcp.*</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_service_address</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__param_target</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9115</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#icmp检查配置</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#C3E88D;">consul_blackbox_icmp</span><span style="color:#89DDFF;">&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">params</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#F07178;">module</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">icmp</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">consul_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">server</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8500</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*blackbox_icmp.*</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_service_address</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__param_target</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">instance</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9115</span></span>
<span class="line"></span>
<span class="line"><span style="color:#676E95;font-style:italic;">#域名检测</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">job_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">consul_domain_exporter</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">scrape_interval</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">10s</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">metrics_path</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/probe</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">consul_sd_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">server</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:8500</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">services</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[]</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*domain.*</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_service_address</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__param_target</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">target_label</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__address__</span></span>
<span class="line"><span style="color:#A6ACCD;">        </span><span style="color:#F07178;">replacement</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">192.168.88.101:9222</span></span></code></pre></div><p>通过 consul_sd_configs 配置⽤于⾃动发现的 Consul 服务地址，服务名为[]，我们通过relabel_configs的过滤规则只接收指定的exporter</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9090/-/reload</span></span></code></pre></div><p>配置完成后重新启动 Prometheus，然后重新查看 Prometheus ⻚⾯上的 targets ⻚⾯，验证上⾯的配置是否存在：<a href="http://192.168.88.101:9090/targets" target="_blank" rel="noreferrer">http://192.168.88.101:9090/targets</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301840239.png" alt="image-20230430184012096" style="zoom:67%;"><h3 id="创建添加脚本" tabindex="-1">创建添加脚本 <a class="header-anchor" href="#创建添加脚本" aria-label="Permalink to &quot;创建添加脚本&quot;">​</a></h3><p>使⽤预先准备好的脚本，⼀次添加多个targets:</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/consul</span></span>
<span class="line"><span style="color:#FFCB6B;">cat</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&gt;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">api.sh</span><span style="color:#89DDFF;">&lt;&lt;</span><span style="color:#A6ACCD;">&quot;</span><span style="color:#89DDFF;">EOF</span><span style="color:#A6ACCD;">&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;node1&quot;,&quot;name&quot;: &quot;node_exporter&quot;,&quot;address&quot;: &quot;node_exporter&quot;,&quot;port&quot;: 9100,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;node_exporter&quot;,&quot;instance&quot;: &quot;Prometheus服务器&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.61:9100/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">curl --request PUT --data @node_exporter.json :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#nginx</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;nginx1&quot;,&quot;name&quot;: &quot;nginx_exporter&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 9113,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;nginx_exporter&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9113/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#rabbitmq</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;rabbitmq1&quot;,&quot;name&quot;: &quot;rabbitmq_exporter&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 9419,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;rabbitmq_exporter&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9419/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#redis</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;redis1&quot;,&quot;name&quot;: &quot;redis_exporter&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 9121,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;redis_exporter&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9121/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#mongodb</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;mongodb1&quot;,&quot;name&quot;: &quot;mongodb_exporter&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 9216,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;mongodb_exporter&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9216/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#mysql</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;mysql1&quot;,&quot;name&quot;: &quot;mysqld_exporter&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 9104,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;mysqld_exporter&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9104/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#cadvisor</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;cadvisor1&quot;,&quot;name&quot;: &quot;cadvisor&quot;,&quot;address&quot;: &quot;cadvisor&quot;,&quot;port&quot;: 8080,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;cadvisor&quot;,&quot;instance&quot;: &quot;Prometheus服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.61:8080/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;cadvisor2&quot;,&quot;name&quot;: &quot;cadvisor&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 8080,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;cadvisor&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:8080/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#springboot</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;springboot1&quot;,&quot;name&quot;: &quot;springboot&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 8081,&quot;tags&quot;: [&quot;springboot&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;springboot&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:8081/actuator/prometheus&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#process_exporter</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;process1&quot;,&quot;name&quot;: &quot;process_exporter&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;port&quot;: 9256,&quot;tags&quot;: [&quot;exporter&quot;],&quot;meta&quot;: {&quot;job&quot;: &quot;process_exporter&quot;,&quot;instance&quot;: &quot;test服务器&quot;,&quot;env&quot;:&quot;test&quot;},&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9256/metrics&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#http</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;http1&quot;,&quot;name&quot;: &quot;blackbox_http&quot;,&quot;address&quot;: &quot;https://www.jd.com&quot;,&quot;tags&quot;: [&quot;blackbox_http&quot;],&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9115&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#tcp</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;tcp1&quot;,&quot;name&quot;: &quot;blackbox_tcp&quot;,&quot;address&quot;: &quot;192.168.11.61:9090&quot;,&quot;tags&quot;: [&quot;blackbox_tcp&quot;],&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9115&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#icmp</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;icmp1&quot;,&quot;name&quot;: &quot;blackbox_icmp&quot;,&quot;address&quot;: &quot;192.168.11.62&quot;,&quot;tags&quot;: [&quot;blackbox_icmp&quot;],&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9115&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"></span>
<span class="line"></span>
<span class="line"><span style="color:#C3E88D;">#domin</span></span>
<span class="line"><span style="color:#C3E88D;">curl -X PUT -d &#39;{&quot;id&quot;: &quot;domain1&quot;,&quot;name&quot;: &quot;domain_exporter&quot;,&quot;address&quot;: &quot;baidu.com&quot;,&quot;tags&quot;: [&quot;domain&quot;],&quot;checks&quot;: [{&quot;http&quot;: &quot;http://192.168.11.62:9222&quot;, &quot;interval&quot;: &quot;5s&quot;}]}&#39;  :8500/v1/agent/service/register</span></span>
<span class="line"><span style="color:#89DDFF;">EOF</span></span></code></pre></div><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">bash api.sh</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304302046520.png" alt="image-20230430204601376" style="zoom:67%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304302046363.png" alt="image-20230430204617137" style="zoom:67%;"><h3 id="consul删除服务" tabindex="-1"><strong>consul</strong>删除服务 <a class="header-anchor" href="#consul删除服务" aria-label="Permalink to &quot;**consul**删除服务&quot;">​</a></h3><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># 注意ID是上面的sh文件对应的id，不是name</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--request</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">PUT</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://127.0.0.1:8500/v1/agent/service/deregister/ID</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--request</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">PUT</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://127.0.0.1:8500/v1/agent/service/deregister/nginx1</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--request</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">PUT</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">http://127.0.0.1:8500/v1/agent/service/deregister/rabbitmq1</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304302049199.png" alt="image-20230430204923053" style="zoom:67%;"><h3 id="存在问题" tabindex="-1">存在问题 <a class="header-anchor" href="#存在问题" aria-label="Permalink to &quot;存在问题&quot;">​</a></h3><p>consul健康检查失败如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301841335.png" alt="image-20230430184127205" style="zoom:80%;"><p>原因：是因为并没有把8080映射出来（下图显示），导致consul监控检查不通过，所以报错。</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301841564.png" alt="image-20230430184144453" style="zoom:80%;"><p>解决：修改 docker-compose.yaml ⽂件把8080端⼝映射出来，就好了，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301842245.png" alt="image-20230430184207075" style="zoom:67%;"><p>修改完成后，执⾏命令，docker-compose up -d</p><h3 id="consulmanager" tabindex="-1">ConsulManager <a class="header-anchor" href="#consulmanager" aria-label="Permalink to &quot;ConsulManager&quot;">​</a></h3><blockquote><p>可以通过它进行可视化管理Consul，无需写命令了</p></blockquote><p><a href="https://gitee.com/starsl/ConsulManager" target="_blank" rel="noreferrer">https://gitee.com/starsl/ConsulManager</a></p><p>1、ConsulManager需要依赖 Consul ，请先完成Consul的部署。（暂时最⾼⽀持Consul v1.14.5）<a href="https://github.com/starsliao/ConsulManager/blob/main/docs/Consul%E9%83%A8%E7%BD%B2%E8%AF%B4%E6%98%8E.md" target="_blank" rel="noreferrer">部署说明</a></p><p>2、使⽤ docker-compose 来部署ConsulManager下载： （仓库根⽬录下 docker-compose.yml ）</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">wget https://starsl.cn/static/img/docker-compose.yml </span></span>
<span class="line"><span style="color:#A6ACCD;">vim docker-compose.yml</span></span></code></pre></div><p>修改3个环境变量：</p><p><strong>consul_token</strong> ：consul的登录token（<a href="https://github.com/starsliao/ConsulManager/blob/main/docs/Consul%E9%83%A8%E7%BD%B2%E8%AF%B4%E6%98%8E.md#%E8%8E%B7%E5%8F%96%E7%99%BB%E5%BD%95token%E8%AE%B0%E5%BD%95secretid%E5%8D%B3%E4%B8%BAconsul%E7%99%BB%E5%BD%95%E7%9A%84token" target="_blank" rel="noreferrer">如何获取</a>？）,当然也可以不获取token，这样consul使⽤⽆密码登录（不安全）。</p><p><strong>consul_url</strong> ：consul的URL(http开头，/v1要保留)</p><p><strong>admin_passwd</strong> ：登录ConsulManager Web的admin密码启动： docker-compose pull &amp;&amp; docker-compose up -d</p><p>访问： http://{IP}:1026 ，使⽤配置的变量 <strong>admin_passwd</strong> 登录安装使⽤中遇到问题，请参考：<a href="https://github.com/starsliao/ConsulManager/blob/main/docs/FAQ.md" target="_blank" rel="noreferrer"><strong>FAQ</strong></a></p><h2 id="relabeling机制" tabindex="-1">Relabeling机制 <a class="header-anchor" href="#relabeling机制" aria-label="Permalink to &quot;Relabeling机制&quot;">​</a></h2><blockquote><p>前⾯讲了Prometheus的⼏种服务发现机制。通过服务发现的⽅式，可以在不重启Prometheus服务的情况下动态的发现需要监控的Target实例信息。</p></blockquote><blockquote><p>基于consul服务发现。 如何加lable标签？Prometheus能够按照某些规则（⽐如标签）从服务发现注册中⼼返回的Target实例中有选择性的采集某些Exporter实例的监控数据。接下来，我们将学习如何通过Prometheus强⼤的Relabel机制来实现以上这些具体的⽬标。</p></blockquote><h3 id="relabeling机制-1" tabindex="-1">Relabeling机制 <a class="header-anchor" href="#relabeling机制-1" aria-label="Permalink to &quot;Relabeling机制&quot;">​</a></h3><h4 id="什么是relabeling机制" tabindex="-1">什么是Relabeling机制？ <a class="header-anchor" href="#什么是relabeling机制" aria-label="Permalink to &quot;什么是Relabeling机制？&quot;">​</a></h4><blockquote><p>在采集样本数据之前，对Target实例的标签(Metadata)进⾏重写的机制在Prometheus被称为Relabeling。Relabeling最基本的应⽤场景就是基于Target实例中包含的metadata标签，动态的添加或者覆盖标签。在采集任务设置中通过relabel_configs来添加⾃定义的Relabeling过程。如下图</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301846692.png" alt="image-20230430184608565" style="zoom:80%;"><blockquote><p>注：细⼼的同学可能发现了，在之前的课程中⿊盒、域名的监控也配置过relabel_configs因为要对Metadata标签进⾏重写，所以我们先来看下Metadata标签</p></blockquote><h4 id="默认metadata标签" tabindex="-1">默认Metadata标签 <a class="header-anchor" href="#默认metadata标签" aria-label="Permalink to &quot;默认Metadata标签&quot;">​</a></h4><blockquote><p>在Prometheus所有的Target实例中，都包含⼀些默认的Metadata标签信息。可以通过Prometheus UI的Targets⻚⾯中查看这些实例的Metadata标签的内容：</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301847218.png" alt="image-20230430184701080" style="zoom:80%;"><p>默认情况下，当Prometheus加载Target实例完成后，这些Target时候都会包含⼀些默认的标签：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301847048.png" alt="image-20230430184719920" style="zoom:67%;"><p>不过这⾥有⼀些例外，例如，我们会发现所有通过Prometheus采集的样本数据中都会包含⼀个名为instance的标签，该标签的内容对应到Target实例的 <strong>address</strong> 。 这⾥实际上是发⽣了⼀次标签的重写处理。</p><h4 id="consul动态发现的metadata标签" tabindex="-1"><strong>Consul</strong>动态发现的<strong>metadata</strong>标签 <a class="header-anchor" href="#consul动态发现的metadata标签" aria-label="Permalink to &quot;**Consul**动态发现的**metadata**标签&quot;">​</a></h4><p>通过Consul动态发现的服务实例还会包含以下Metadata标签信息：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301848784.png" alt="image-20230430184800660" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301848169.png" alt="image-20230430184810038" style="zoom:80%;"><p>如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301848527.png" alt="image-20230430184831359" style="zoom:80%;"><h3 id="relabel-configs配置" tabindex="-1">relabel_configs配置 <a class="header-anchor" href="#relabel-configs配置" aria-label="Permalink to &quot;relabel_configs配置&quot;">​</a></h3><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301849651.png" alt="image-20230430184909518" style="zoom:67%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301849622.png" alt="image-20230430184922495" style="zoom:67%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202304301849284.png" style="zoom:80%;"><p>relabel_configs的配置我们已经知道了是对源数据重写的配置，接下来我们就来看具体案例</p><h4 id="替换标签值" tabindex="-1">替换标签值 <a class="header-anchor" href="#替换标签值" aria-label="Permalink to &quot;替换标签值&quot;">​</a></h4><p>可以通过 replace 这个操作来完成，如果没有指定 action 字段，则默认就是 replace 。</p><p>注：源数据需要加 [] 以便区分。⽬标数据不需要加。</p><p>案例1</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">relabel_configs:</span></span>
<span class="line"><span style="color:#A6ACCD;"> - source_labels: [__meta_consul_service_address]</span></span>
<span class="line"><span style="color:#A6ACCD;"> target_label: __param_target</span></span>
<span class="line"><span style="color:#A6ACCD;"> - source_labels: [__param_target]</span></span>
<span class="line"><span style="color:#A6ACCD;"> target_label: instance</span></span>
<span class="line"><span style="color:#A6ACCD;"> - target_label: __address__</span></span>
<span class="line"><span style="color:#A6ACCD;"> replacement: 192.168.11.62:9115</span></span></code></pre></div><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">源数据：__meta_consul_service_address=&quot;192.168.11.62&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">会变成：target=&quot;192.168.11.62&quot;,instance=&quot;192.168.11.62&quot;,endpoint=&quot;192.168.11.62:9115/prod&quot;</span></span></code></pre></div><p>案例2</p><p>从Node Exporter实例采集上来的样本数据如下所示：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">node_load1{instance=&quot;test服务器&quot;, job=&quot;node_exporter&quot;} 0.04</span></span></code></pre></div><p>我们希望能有⼀个额外的标签dc，可以表示该样本所属的数据中⼼：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">node_load1{instance=&quot;test服务器&quot;, job=&quot;node_exporter&quot;,dc=&quot;dc1&quot;} 0.04</span></span></code></pre></div><p>relabel_config配置：</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">relabel_configs:</span></span>
<span class="line"><span style="color:#A6ACCD;">  - source_labels: [&quot;__meta_consul_dc&quot;]</span></span>
<span class="line"><span style="color:#A6ACCD;">    target_label: &quot;dc&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;"></span></span>
<span class="line"><span style="color:#A6ACCD;">源数据：__meta_consul_dc=&quot;dc1&quot;会变成：dc=&quot;dc1&quot;</span></span></code></pre></div><h4 id="保留或丢弃对象" tabindex="-1">保留或丢弃对象 <a class="header-anchor" href="#保留或丢弃对象" aria-label="Permalink to &quot;保留或丢弃对象&quot;">​</a></h4><p>通过 keep 或 drop 这两个动作可以来完成</p><p>案例1</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">source_labels</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#C3E88D;">__meta_consul_tags</span><span style="color:#89DDFF;">]</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">.*exporter.*</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">keep</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#F07178;">源数据为：__meta_consul_tags=&quot;,exporter,&quot; 会保留。如果action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">drop就相反</span></span>
<span class="line"><span style="color:#F07178;">源数据为：__meta_consul_tags=&quot;,domain,&quot; 会丢弃，如果action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">drop就相反</span></span></code></pre></div><h4 id="标签集映射" tabindex="-1">标签集映射 <a class="header-anchor" href="#标签集映射" aria-label="Permalink to &quot;标签集映射&quot;">​</a></h4><p>把源标签的值映射到⼀组新的标签中去，使⽤ labelmap 这个动作完成</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">__meta_consul_service_metadata_(.+)</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelmap</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#F07178;">源数据为：__meta_consul_service_metadata_job=&quot;test&quot;会变成</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job=&quot;test&quot;</span></span></code></pre></div><h4 id="保留或删除标签" tabindex="-1">保留或删除标签 <a class="header-anchor" href="#保留或删除标签" aria-label="Permalink to &quot;保留或删除标签&quot;">​</a></h4><p>使⽤ labelkeep 和 labeldrop 这两个操作</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">relabel_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">regex</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">job</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labeldrop</span></span>
<span class="line"><span style="color:#A6ACCD;"> </span></span>
<span class="line"><span style="color:#C3E88D;">数据：probe_success{instance=&quot;192.168.11.61:9090&quot;, job=&quot;consul_blackbox_tcp&quot;}</span></span>
<span class="line"><span style="color:#F07178;">变成：probe_success{instance=&quot;192.168.11.61:9090&quot;} action</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">labelkeep 刚好相反</span></span></code></pre></div><h1 id="高可用" tabindex="-1">高可用 <a class="header-anchor" href="#高可用" aria-label="Permalink to &quot;高可用&quot;">​</a></h1><h2 id="thanos" tabindex="-1">thanos <a class="header-anchor" href="#thanos" aria-label="Permalink to &quot;thanos&quot;">​</a></h2><blockquote><p>Prometheus在小规模的部署中，完全不需要依赖其他组件就可以达到监控的目的。但在应对大规模、大数据量的集群时，就存在缺少集群、数据可靠性保障的支持。Prometheus虽然支持联邦部署模式，但这个架构还是会有其他问题，如数据会被重复储存在两个地方，还有被拉取的Prometheus机器有可能发生超时现象。另外，监控数据分散在多台Prometheus监控节点，在变动或者长期存储时依赖本地磁盘，本地磁盘一般不是高可用的存储，监控数据就变得没有可靠性保障。为了解决上述问题，产生了多种解决方案，Thanos方案是最成熟、广泛使用的。</p></blockquote><h3 id="thanos概述" tabindex="-1">thanos概述 <a class="header-anchor" href="#thanos概述" aria-label="Permalink to &quot;thanos概述&quot;">​</a></h3><blockquote><p><a href="https://thanos.io/" target="_blank" rel="noreferrer">Thanos</a>是一个基于 Prometheus 实现的监控方案，其主要设计目的是解决原生 Prometheus 上的痛点，并且做进一步的提升，主要的特性有：<strong>全局查询，高可用，动态拓展，长期存储</strong>。</p></blockquote><h3 id="thanos架构" tabindex="-1">thanos架构 <a class="header-anchor" href="#thanos架构" aria-label="Permalink to &quot;thanos架构&quot;">​</a></h3><p>Thanos 主要由如下几个特定功能的组件组成：</p><blockquote><p>边车组件（Sidecar）：连接到 Prometheus，并把 Prometheus 暴露给查询网关（Querier/Query），以供实时查询，并且可以上传 Prometheus 数据到云存储，以供长期保存（和Receiver二选一）</p></blockquote><blockquote><p>查询网关（Querier）：实现 Prometheus API 以聚合来自底层组件（如边车组件 Sidecar，或是存储网关 Store Gateway）的数据</p><p>存储网关（Store Gateway）：将云存储中的数据内容暴露出来（可选）</p><p>压缩器（Compactor）：将云存储中的数据进行压缩和下采样和保留（可选）接收器（Receiver）：从 Prometheus 的远程写入 WAL 接收数据，将其暴露出去或者上传到云存储（和Sidecar二选一）</p></blockquote><blockquote><p>规则组件（Ruler）：根据 Thanos 中的数据评估记录和警报规则（可选）</p><p>查询前端：实现 Prometheus 的 API，将其代理给 Query，同时缓存响应（可选）</p><p>因为用到thanos Sidecar和thanos Querier接下来我们就来安装。</p></blockquote><h2 id="prometheus高可用" tabindex="-1">Prometheus高可用 <a class="header-anchor" href="#prometheus高可用" aria-label="Permalink to &quot;Prometheus高可用&quot;">​</a></h2><h3 id="基本安装" tabindex="-1">基本安装 <a class="header-anchor" href="#基本安装" aria-label="Permalink to &quot;基本安装&quot;">​</a></h3><blockquote><p>三台机器分别安装docker和docker-compose，三台机器分别是node1,node2,node3，就使用最上面的安装即可</p></blockquote><blockquote><p>node1和node2安装prometheus，就使用上面的docker版本安装即可</p></blockquote><blockquote><p>因为Thanos Querier需要调用Prometheus API 来查询数据，所以prometheus需要开启api功能，就在Gitee拉取的docker-compose.yml文件里，去掉注释即可，node1和node2都要配置</p></blockquote><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011345763.png" alt="image-20230501134553675" style="zoom:67%;"><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 重新启动</span></span>
<span class="line"><span style="color:#FFCB6B;">docker-compose</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">up</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span></span></code></pre></div><h3 id="thanos安装" tabindex="-1">Thanos安装 <a class="header-anchor" href="#thanos安装" aria-label="Permalink to &quot;Thanos安装&quot;">​</a></h3><blockquote><p>node1和node2都要配置</p></blockquote><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">prometheus/prometheus.yml</span></span></code></pre></div><p>增加如下配置：</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">global</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#89DDFF;">  </span><span style="color:#676E95;font-style:italic;"># 与外部系统通信时对时间序列或者告警信息添加的标签</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">external_labels</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">cluster</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">linge</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><p>完成后重置prometheus</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">curl -X POST :9090/-/reload</span></span></code></pre></div><p>安装Thanos Sidecar和 Thanos Querier</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus/</span></span>
<span class="line"><span style="color:#FFCB6B;">cp</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker-compose.yaml{,.bak1}</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker-compose.yaml</span></span></code></pre></div><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">thanos-sidecar</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">thanosio/thanos:v0.31.0</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">thanos-sidecar</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">sidecar</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--http-address=0.0.0.0:19191</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--grpc-address=0.0.0.0:19090</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--tsdb.path=prometheus_data</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--prometheus.url=http://192.168.88.101:9090</span><span style="color:#A6ACCD;"> </span><span style="color:#676E95;font-style:italic;">#192.168.11.60 node1上使用这个配置</span></span>
<span class="line"><span style="color:#89DDFF;">      </span><span style="color:#676E95;font-style:italic;"># - --prometheus.url=http://192.168.88.102:9090 #192.168.11.61 node2上使用这个配置</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">19090:19090</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">19191:19191</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">thanos-query</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">image</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">thanosio/thanos:v0.31.0</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">container_name</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">thanos-query</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">restart</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">always</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">query</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--http-address=0.0.0.0:19192</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--grpc-address=0.0.0.0:19092</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--query.replica-label=prometheus_replica</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--store=192.168.88.101:19090</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">--store=192.168.88.102:19090</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">19192:19192</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">19092:19092</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><p>访问测试</p><p><a href="http://192.168.88.101:19192/stores" target="_blank" rel="noreferrer">http://192.168.88.101:19192/stores</a></p><p><a href="http://192.168.88.102:19192/stores" target="_blank" rel="noreferrer">http://192.168.88.102:19192/stores</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011410416.png" alt="image-20230501141036291" style="zoom:80%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011411329.png" alt="image-20230501141104209" style="zoom:80%;"><h3 id="grafana修改数据源" tabindex="-1">Grafana修改数据源 <a class="header-anchor" href="#grafana修改数据源" aria-label="Permalink to &quot;Grafana修改数据源&quot;">​</a></h3><p><a href="http://192.168.88.101:3000/" target="_blank" rel="noreferrer">http://192.168.88.101:3000/</a> 用户admin 密码password</p><p>url地址填写： <a href="http://192.168.88.101:19192" target="_blank" rel="noreferrer">http://192.168.88.101:19192</a></p><p><a href="http://192.168.88.102:3000/" target="_blank" rel="noreferrer">http://192.168.88.102:3000/</a> 用户admin 密码password</p><p>url地址填写： <a href="http://192.168.88.102:19192" target="_blank" rel="noreferrer">http://192.168.88.102:19192</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011413796.png" alt="image-20230501141358654" style="zoom:67%;"><p>添加dashboard： 1860，至此prometheus的高可用就完成了</p><h3 id="thanos远程存储-可选" tabindex="-1">Thanos远程存储（可选） <a class="header-anchor" href="#thanos远程存储-可选" aria-label="Permalink to &quot;Thanos远程存储（可选）&quot;">​</a></h3><p>默认prometheus的数据存储受 --storage.tsdb.retention.time=30d 控制，可以使用Thanos把prometheus的数据永久保存</p><p><a href="https://thanos.io/tip/thanos/storage.md/" target="_blank" rel="noreferrer">https://thanos.io/tip/thanos/storage.md/</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011415180.png" alt="image-20230501141508050" style="zoom:67%;"><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011415979.png" alt="image-20230501141521859" style="zoom:80%;"><blockquote><p>要在生产环境使用最好使用 Stable 状态的，比如 S3 或者兼容 S3 的服务，比如 Ceph、Minio 等等。对于国内用户当然最方便的还是直接使用阿里云 OSS 或者腾讯云 COS 这样的服务，但是很多时候可能我们的服务并不是跑在公有云上面的，所以这里我们用 Minio 来部署一个兼容 S3 协议的对象存储服务。</p></blockquote><p>thanos.yaml配置，minio配置</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">type: s3</span></span>
<span class="line"><span style="color:#A6ACCD;">  config:</span></span>
<span class="line"><span style="color:#A6ACCD;">    bucket: &quot;&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    endpoint: ip:9000</span></span>
<span class="line"><span style="color:#A6ACCD;">    access_key: &quot;&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    secret_key: &quot;&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    insecure: true</span></span></code></pre></div><p>阿里云oss配置：thanos.yaml</p><div class="language-tml"><button title="Copy Code" class="copy"></button><span class="lang">tml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">type: ALIYUNOSS</span></span>
<span class="line"><span style="color:#A6ACCD;">  config:</span></span>
<span class="line"><span style="color:#A6ACCD;">    endpoint: &quot;&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    bucket: &quot;&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    access_key_id: &quot;&quot;</span></span>
<span class="line"><span style="color:#A6ACCD;">    access_key_secret: &quot;&quot;</span></span></code></pre></div><p><strong>thanos-sidecar配置远程存储</strong></p><p>Thanos远程存储是通过配置Sidecar读取Prometheus收集的数据，并写入到远程的对象存储中。所以在Sidecar中增加如下配置：注：配置远程存储后，需通过thanos Store Gateway来查看远程存储数据</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">--objstore.config-file</span><span style="color:#A6ACCD;">=/etc/thanos/thanos.yaml</span></span></code></pre></div><h2 id="alertmanager高可用" tabindex="-1">alertmanager高可用 <a class="header-anchor" href="#alertmanager高可用" aria-label="Permalink to &quot;alertmanager高可用&quot;">​</a></h2><p>在上面我们已经在prometheus01和prometheus02上都安装了alertmanager。一共部署了2套alertmanager，那是不是就已经有高可用了？我们先来测试下：</p><p>模拟发送告警信息，2台机器上都执行，直接粘贴到命令行执行即可</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;">#!/usr/bin/env bash</span></span>
<span class="line"></span>
<span class="line"><span style="color:#A6ACCD;">alerts1</span><span style="color:#89DDFF;">=</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">[</span></span>
<span class="line"><span style="color:#C3E88D;">    {</span></span>
<span class="line"><span style="color:#C3E88D;">        &quot;labels&quot;: {</span></span>
<span class="line"><span style="color:#C3E88D;">            &quot;alertname&quot;: &quot;DiskRunningFull&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">            &quot;dev&quot;: &quot;sda1&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">            &quot;instance&quot;: &quot;example1&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">            &quot;severity&quot;: &quot;critical&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">        },</span></span>
<span class="line"><span style="color:#C3E88D;">        &quot;annotations&quot;: {</span></span>
<span class="line"><span style="color:#C3E88D;">            &quot;description&quot;: &quot;The disk sda1 is running full&quot;,</span></span>
<span class="line"><span style="color:#C3E88D;">            &quot;summary&quot;: &quot;please check the instance example1&quot;</span></span>
<span class="line"><span style="color:#C3E88D;">        }</span></span>
<span class="line"><span style="color:#C3E88D;">    }</span></span>
<span class="line"><span style="color:#C3E88D;">]</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-XPOST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-d</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;">$alerts1</span><span style="color:#89DDFF;">&quot;</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9093/api/v1/alerts</span></span></code></pre></div><p><a href="http://192.168.88.101:9093/#/alerts" target="_blank" rel="noreferrer">http://192.168.88.101:9093/#/alerts</a></p><p><a href="http://192.168.88.102:9093/#/alerts" target="_blank" rel="noreferrer">http://192.168.88.102:9093/#/alerts</a></p><blockquote><p>同一台告警信息，发送了2次。由于ALertmanager之间不存在并不了解彼此的存在，因此则会出现告警通知被不同的Alertmanager重复，发送多次的问题。</p></blockquote><blockquote><p>4为了解决这一问题，Alertmanager在集群配置中引入了Gossip机制，Gossip机制为多个Alertmanager之间提供了信息传递的机制。确保及时在多个Alertmanager分别接收到相同告警信息的情况下，也只有一个告警通知被发送给Receiver。</p></blockquote><p>node1服务器</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker-compose.yaml</span></span></code></pre></div><p>增加如下配置，注意是在原alertmanager配置里增加</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">alertmanager</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--log.level=debug</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--cluster.listen-address=0.0.0.0:8001</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">8001:8001</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011453957.png" alt="image-20230501145342841" style="zoom:80%;"><p>如果是二进制安装的alertmanager，只需要增加一行配置: --cluster.listen-address=0.0.0.0:8001,应用配置</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">docker-compose up -d</span></span></code></pre></div><p>node2服务器</p><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#82AAFF;">cd</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">/data/docker-prometheus</span></span>
<span class="line"><span style="color:#FFCB6B;">vim</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">docker-compose.yaml</span></span></code></pre></div><p>增加如下配置</p><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#F07178;">alertmanager</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">command</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--log.level=debug</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--cluster.listen-address=0.0.0.0:8002</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">--cluster.peer=192.168.88.101:8001</span><span style="color:#89DDFF;">&#39;</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#F07178;">ports</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">      </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">8002:8002</span><span style="color:#89DDFF;">&#39;</span></span></code></pre></div><p>如果是二进制安装的alertmanager，只需要增加2行配置: --cluster.listen-address=0.0.0.0:8001 和 --</p><p>cluster.peer=192.168.11.60:8001</p><div class="language-"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#A6ACCD;">docker-compose up -d</span></span></code></pre></div><p><a href="http://192.168.88.101:9093/#/status" target="_blank" rel="noreferrer">http://192.168.88.101:9093/#/status</a></p><p><a href="http://192.168.88.102:9093/#/status" target="_blank" rel="noreferrer">http://192.168.88.102:9093/#/status</a></p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011429348.png" alt="image-20230501142954231" style="zoom:67%;"><p>当集群配置完成后，会自带Gossip机制，接下来我们在测试下，同一台告警信息只发送了一次</p><h2 id="prometheus和alertmanager集群整合" tabindex="-1">Prometheus和alertmanager集群整合 <a class="header-anchor" href="#prometheus和alertmanager集群整合" aria-label="Permalink to &quot;Prometheus和alertmanager集群整合&quot;">​</a></h2><blockquote><p>由于实现了Gossip机制，在Prometheus和Alertmanager实例之间不要使用任何负载均衡，需要确保Prometheus将告警发送到所有的Alertmanager实例中2台prometheus修改配置如下，2台都要操作。</p></blockquote><div class="language-yml"><button title="Copy Code" class="copy"></button><span class="lang">yml</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#676E95;font-style:italic;"># Alertmanager 配置</span></span>
<span class="line"><span style="color:#F07178;">alerting</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#F07178;">alertmanagers</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">  </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">static_configs</span><span style="color:#89DDFF;">:</span></span>
<span class="line"><span style="color:#A6ACCD;">    </span><span style="color:#89DDFF;">-</span><span style="color:#A6ACCD;"> </span><span style="color:#F07178;">targets</span><span style="color:#89DDFF;">:</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">[</span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.101:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">,</span><span style="color:#A6ACCD;"> </span><span style="color:#89DDFF;">&#39;</span><span style="color:#C3E88D;">192.168.88.102:9093</span><span style="color:#89DDFF;">&#39;</span><span style="color:#89DDFF;">]</span></span></code></pre></div><div class="language-sh"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki material-theme-palenight"><code><span class="line"><span style="color:#FFCB6B;">curl</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">-X</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">POST</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">:9090/-/reload</span></span>
<span class="line"><span style="color:#676E95;font-style:italic;"># 再次测试</span></span>
<span class="line"><span style="color:#FFCB6B;">docker</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">stop</span><span style="color:#A6ACCD;"> </span><span style="color:#C3E88D;">node-exporter</span></span></code></pre></div><h2 id="停机测试" tabindex="-1">停机测试 <a class="header-anchor" href="#停机测试" aria-label="Permalink to &quot;停机测试&quot;">​</a></h2><p>把prometheus01服务器关闭，打开thanos-query检查，如下图：</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011302652.png" alt="image-20230501130244553" style="zoom:80%;"><p>grafana检查，图形都是正常的。告警通知也正常</p><img src="https://edu-8673.oss-cn-beijing.aliyuncs.com/img2023.3.30/202305011303470.png" alt="image-20230501130302336" style="zoom:80%;"><blockquote><p>通过安装Thanos Sidecar和Thanos Querier完成了prometheus的高可用。通过alertmanager的集群配置（自带Gossip机制）来解决告警重复发送问题。每次修改prometheus配置，添加grafana的Dashboard图形都需要在2台机器上都要操作。可以使用consul，dns等自动发现功能来减少配置带来的麻烦。</p></blockquote></div></div></main><footer class="VPDocFooter" data-v-6b87e69f data-v-37656e44><!--[--><!--]--><!----><nav class="prev-next" data-v-37656e44><div class="pager" data-v-37656e44><a class="pager-link prev" href="/notebook/%E4%BA%91%E5%8E%9F%E7%94%9F/K8S.html" data-v-37656e44><span class="desc" data-v-37656e44>Previous page</span><span class="title" data-v-37656e44>K8S</span></a></div><div class="pager" data-v-37656e44><a class="pager-link next" href="/notebook/%E5%8F%AF%E8%A7%86%E5%8C%96%20&amp;%20%E7%9B%91%E6%8E%A7/%E7%9B%91%E6%8E%A7%E8%BF%9B%E9%98%B6.html" data-v-37656e44><span class="desc" data-v-37656e44>Next page</span><span class="title" data-v-37656e44>监控进阶</span></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
    <script>window.__VP_HASH_MAP__=JSON.parse("{\"2、数据库_mysql_mysql面试_基础.md\":\"40da680a\",\"1、学前端_5、小程序_小程序项目.md\":\"60a1629b\",\"1、学前端_4、node_知识篇.md\":\"a7fb500e\",\"1、学前端_2、js_ts_es6 进阶.md\":\"6d07ba10\",\"1、学前端_3、vue_vue3_vue3进阶.md\":\"7ac622b4\",\"5、运维_jenkins.md\":\"929081f8\",\"1、学前端_2、js_ts_typescript.md\":\"875a4aa4\",\"2、数据库_mysql_mysql核心_设计.md\":\"7faf46d1\",\"2、数据库_mysql_mysql核心_基础.md\":\"d8e97f3e\",\"1、学前端_1、html_css_html基础.md\":\"7584d076\",\"1、学前端_5、专题篇_问题篇.md\":\"e893aaa2\",\"2、数据库_mysql_mysql面试_进阶.md\":\"f934806d\",\"3、springboot_运维_原理.md\":\"f4a39db6\",\"2、数据库_influxdb.md\":\"6e1711e1\",\"3、springboot_新特性.md\":\"cdf3e307\",\"mybatis_mybatisplus_jpa.md\":\"8e41681b\",\"1、学前端_5、小程序_小程序优化.md\":\"a2185198\",\"2、数据库_redis_redis基础.md\":\"856df0e0\",\"linux_实用脚本.md\":\"f2299dd5\",\"4、微服务_必备_分布式基础.md\":\"d49863d5\",\"2、数据库_redis_redis优化.md\":\"e66ae32f\",\"4、微服务_springsecurity_进阶篇.md\":\"235a8e9e\",\"5、运维_chatgpt.md\":\"10db3823\",\"2、数据库_mysql_分库分表.md\":\"e1c8a095\",\"start.md\":\"9bc1ff8d\",\"5、运维_github.md\":\"2ec6c735\",\"java学前端_css.md\":\"f11b47f0\",\"1、学前端_5、专题篇_知识篇.md\":\"a463ed8d\",\"linux_软件部署.md\":\"d6722925\",\"2、数据库_neo4j.md\":\"97ad22ac\",\"team.md\":\"ce467a6a\",\"nginx_实战篇.md\":\"7785486e\",\"index.md\":\"8c3ec167\",\"计算机基础_计算机网络_网络基础.md\":\"7a54a85d\",\"1、学前端_4、node_进阶篇.md\":\"60f6db69\",\"java_java集合.md\":\"a049b313\",\"1、学前端_3、vue_vue3_vue3高级.md\":\"614d1516\",\"1、学前端_5、小程序_微信小程序.md\":\"9a4be771\",\"5、运维_netty.md\":\"12ca0278\",\"2、数据库_mysql_mysql核心_运维.md\":\"83f97c16\",\"idea_vs code.md\":\"afdcb593\",\"java学前端_vue3_组件.md\":\"1086884e\",\"idea_chrome.md\":\"4a32afbc\",\"云原生_k8s.md\":\"db58e65a\",\"2、数据库_mysql_mysql核心_进阶.md\":\"61d16dff\",\"ssm_springbatch.md\":\"f799ab4a\",\"三高_分布式.md\":\"db1b8a1b\",\"2、数据库_elasticsearch_1、es基础.md\":\"04d17448\",\"linux_linux基础.md\":\"4b0bf394\",\"idea_idea插件.md\":\"fa86e45a\",\"可视化 _ 监控_可视化大屏.md\":\"004553bd\",\"2、数据库_mongodb_整合.md\":\"3c47d7f4\",\"4、微服务_springsecurity_基础篇.md\":\"534a3401\",\"4、微服务_进阶.md\":\"69095c58\",\"计算机基础_计算机基础_操作系统.md\":\"0f75d113\",\"可视化 _ 监控_zabbix.md\":\"71f2270e\",\"nginx_基础篇.md\":\"c7d8bb50\",\"1、学前端_4、node_项目实战.md\":\"bc5065b8\",\"2、数据库_redis_redis原理.md\":\"5cedf685\",\"可视化 _ 监控_监控基础.md\":\"ac56ce4d\",\"三高_高并发.md\":\"ea9ffc99\",\"2、数据库_redis_redis高级.md\":\"1d5872f6\",\"1、学前端_4、node_基础篇.md\":\"581cc13a\",\"2、数据库_mongodb_基础.md\":\"fb7a0a29\",\"idea_idea基础.md\":\"6f2f9638\",\"4、微服务_必备_sentinel.md\":\"2edfbf6c\",\"2、数据库_elasticsearch_3、es高级.md\":\"ef146606\",\"1、学前端_3、vue_vue3_vue3新语法.md\":\"8afd5409\",\"消息中间件_canal.md\":\"3949163c\",\"ssm_maven.md\":\"2c5e12ed\",\"4、微服务_springsecurity_高级篇.md\":\"882d3ff3\",\"linux_linux进阶.md\":\"188ef7b4\",\"计算机基础_设计模式_uml.md\":\"634ba256\",\"计算机基础_算法_leetcode.md\":\"77162fb9\",\"项目实战_小兔鲜_进阶篇1.md\":\"17c52c81\",\"1、学前端_2、js_ts_es6 基础.md\":\"fda3f18b\",\"项目实战_小兔鲜_进阶篇2.md\":\"a0f23006\",\"软件测试_测试基础.md\":\"8c1060cd\",\"2、数据库_redis_本地缓存.md\":\"00617fe6\",\"nginx_面试篇.md\":\"e3fb373a\",\"mybatis_mybatisplus_mybatis.md\":\"9239e0ad\",\"linux_shell.md\":\"ae53d83b\",\"2、数据库_mysql_mysql核心_优化.md\":\"36230425\",\"项目实战_项目推荐.md\":\"f9d97630\",\"mybatis_mybatisplus_mybatisplus.md\":\"0030fd35\",\"项目实战_百度地图_进阶篇.md\":\"c8b93267\",\"三高_高可用.md\":\"323840c5\",\"java_java新特性.md\":\"22abf56d\",\"软件测试_压力测试.md\":\"9ab44440\",\"java学前端_html_js.md\":\"e0fcd240\",\"2、数据库_redis_redis实战.md\":\"d6daeeab\",\"nginx_进阶篇.md\":\"e6b63195\",\"三高_秒杀.md\":\"3878bb64\",\"5、运维_git.md\":\"0264925c\",\"java_java进阶.md\":\"e79cb5b4\",\"并发 _ 多线程_基础篇.md\":\"7adbfac5\",\"项目实战_百度地图_基础篇.md\":\"8afa5954\",\"java学前端_react.md\":\"3ec827dd\",\"1、学前端_1、html_css_css基础.md\":\"01b56712\",\"项目实战_小兔鲜_基础篇.md\":\"646f5df5\",\"1、学前端_2、js_ts_js 基础.md\":\"cb13e36f\",\"可视化 _ 监控_监控进阶.md\":\"0cdbc292\",\"计算机基础_设计模式_基础篇.md\":\"51617287\",\"计算机基础_数据结构_基础篇.md\":\"b2bfd8d4\",\"项目实战_苍穹外卖_进阶篇.md\":\"48415e41\",\"ssm_spring.md\":\"ab514659\",\"消息中间件_rabbitmq.md\":\"45b1eb28\",\"1、学前端_1、html_css_网页进阶.md\":\"db998248\",\"消息中间件_kafka.md\":\"b747dabf\",\"云原生_docker.md\":\"983c7ba7\",\"4、微服务_必备_分布式锁.md\":\"5af1cf8d\",\"消息中间件_rocketmq.md\":\"d441da85\",\"项目实战_黑马头条_基础篇.md\":\"b05af3a6\",\"ssm_springmvc.md\":\"81b9714f\",\"项目实战_支付.md\":\"1d7407dd\",\"项目实战_黑马头条_进阶篇2.md\":\"bff0015b\",\"项目实战_黑马头条_进阶篇.md\":\"19f18388\",\"java学前端_vue2_组件.md\":\"58c6b1df\",\"3、springboot_基础篇.md\":\"529c66f4\",\"3、springboot_应用篇.md\":\"8b92aa61\",\"项目实战_黑马头条_高级篇.md\":\"227c08c1\",\"1、学前端_5、小程序_uniapp.md\":\"71a282b4\",\"项目实战_云尚办公_基础篇.md\":\"1fe188ba\",\"并发 _ 多线程_并发完善.md\":\"26619c46\",\"1、学前端_2、js_ts_js 进阶.md\":\"657dfb8f\",\"java_java高级.md\":\"23782d1a\",\"java_java基础.md\":\"86d67c77\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"VitePress\",\"description\":\"A VitePress site\",\"base\":\"/notebook/\",\"head\":[],\"appearance\":true,\"themeConfig\":{\"algolia\":{\"appId\":\"DW7O63I9IR\",\"apiKey\":\"f8ed758cdb288a8b06542bc35923c1a1\",\"indexName\":\"notebook\"},\"sidebar\":[{\"text\":\"Java\",\"collapsed\":true,\"items\":[{\"text\":\"Java基础\",\"link\":\"/Java/Java基础\"},{\"text\":\"Java新特性\",\"link\":\"/Java/Java新特性\"},{\"text\":\"Java进阶\",\"link\":\"/Java/Java进阶\"},{\"text\":\"Java集合\",\"link\":\"/Java/Java集合\"},{\"text\":\"Java高级\",\"link\":\"/Java/Java高级\"}]},{\"text\":\"Linux\",\"collapsed\":true,\"items\":[{\"text\":\"Linux基础\",\"link\":\"/Linux/Linux基础\"},{\"text\":\"Linux新特性\",\"link\":\"/Linux/Linux进阶\"},{\"text\":\"Shell脚本\",\"link\":\"/Linux/Shell\"},{\"text\":\"实用脚本\",\"link\":\"/Linux/实用脚本\"},{\"text\":\"软件部署\",\"link\":\"/Linux/软件部署\"}]},{\"text\":\"Nginx\",\"collapsed\":true,\"items\":[{\"text\":\"基础篇\",\"link\":\"/Nginx/基础篇\"},{\"text\":\"进阶篇\",\"link\":\"/Nginx/进阶篇\"},{\"text\":\"实战篇\",\"link\":\"/Nginx/实战篇\"},{\"text\":\"面试篇\",\"link\":\"/Nginx/面试篇\"}]},{\"text\":\"SSM\",\"collapsed\":true,\"items\":[{\"text\":\"Maven\",\"link\":\"/SSM/Maven\"},{\"text\":\"Spring\",\"link\":\"/SSM/Spring\"},{\"text\":\"SpringMVC\",\"link\":\"/SSM/SpringMVC\"},{\"text\":\"SpringBatch\",\"link\":\"/SSM/SpringBatch\"}]},{\"text\":\"SpringBoot\",\"collapsed\":true,\"items\":[{\"text\":\"基础篇\",\"link\":\"/3、SpringBoot/基础篇\"},{\"text\":\"应用篇\",\"link\":\"/3、SpringBoot/应用篇\"},{\"text\":\"新特性\",\"link\":\"/3、SpringBoot/新特性\"},{\"text\":\"运维&原理\",\"link\":\"/3、SpringBoot/运维&原理\"}]},{\"text\":\"SpringCloud\",\"collapsed\":true,\"items\":[{\"text\":\"SpringCloud\",\"link\":\"/4、微服务/进阶\"},{\"text\":\"Sentinel\",\"link\":\"/4、微服务/必备/Sentinel\"}]},{\"text\":\"SpringSecurity\",\"collapsed\":true,\"items\":[{\"text\":\"SpringSecurity基础篇\",\"link\":\"/4、微服务/SpringSecurity/基础篇\"},{\"text\":\"SpringSecurity进阶篇\",\"link\":\"/4、微服务/SpringSecurity/进阶篇\"},{\"text\":\"SpringSecurity高级篇\",\"link\":\"/4、微服务/SpringSecurity/高级篇\"}]},{\"text\":\"Mybatis & MybatisPlus\",\"collapsed\":true,\"items\":[{\"text\":\"Mybatis\",\"link\":\"/Mybatis&MybatisPlus/Mybatis\"},{\"text\":\"MybatisPlus\",\"link\":\"/Mybatis&MybatisPlus/MybatisPlus\"},{\"text\":\"JPA\",\"link\":\"/Mybatis&MybatisPlus/JPA\"}]},{\"text\":\"Git & ChatGPT\",\"collapsed\":true,\"items\":[{\"text\":\"Git\",\"link\":\"/5、运维/Git\"},{\"text\":\"Github\",\"link\":\"/5、运维/Github\"},{\"text\":\"ChatGPT\",\"link\":\"/5、运维/ChatGPT\"},{\"text\":\"Jenkins\",\"link\":\"/5、运维/Jenkins\"},{\"text\":\"Netty\",\"link\":\"/5、运维/Netty\"}]},{\"text\":\"数据库\",\"collapsed\":true,\"items\":[{\"text\":\"MySQL\",\"collapsed\":true,\"items\":[{\"text\":\"MySQL基础\",\"link\":\"/2、数据库/MySQL/MySQL核心/基础\"},{\"text\":\"MySQL进阶\",\"link\":\"/2、数据库/MySQL/MySQL核心/进阶\"},{\"text\":\"MySQL优化\",\"link\":\"/2、数据库/MySQL/MySQL核心/优化\"},{\"text\":\"MySQL设计\",\"link\":\"/2、数据库/MySQL/MySQL核心/设计\"},{\"text\":\"MySQL运维\",\"link\":\"/2、数据库/MySQL/MySQL核心/运维\"},{\"text\":\"分库分表\",\"link\":\"/2、数据库/MySQL/分库分表\"}]},{\"text\":\"Redis\",\"collapsed\":true,\"items\":[{\"text\":\"Redis基础\",\"link\":\"/2、数据库/Redis/Redis基础\"},{\"text\":\"Redis优化\",\"link\":\"/2、数据库/Redis/Redis优化\"},{\"text\":\"Redis原理\",\"link\":\"/2、数据库/Redis/Redis原理\"},{\"text\":\"Redis高级\",\"link\":\"/2、数据库/Redis/Redis高级\"},{\"text\":\"Redis实战\",\"link\":\"/2、数据库/Redis/Redis实战\"},{\"text\":\"本地缓存\",\"link\":\"/2、数据库/Redis/本地缓存\"}]},{\"text\":\"MongoDB\",\"collapsed\":true,\"items\":[{\"text\":\"MongoDB基础\",\"link\":\"/2、数据库/MongoDB/基础\"},{\"text\":\"MongoDB进阶\",\"link\":\"/2、数据库/MongoDB/整合\"}]},{\"text\":\"ElasticSearch\",\"collapsed\":true,\"items\":[{\"text\":\"ES基础\",\"link\":\"/2、数据库/ElasticSearch/1、ES基础\"},{\"text\":\"ES高级\",\"link\":\"/2、数据库/ElasticSearch/3、ES高级\"}]},{\"text\":\"InfluxDB\",\"link\":\"/2、数据库/influxdb\"},{\"text\":\"Neo4j\",\"link\":\"/2、数据库/Neo4j\"}]},{\"text\":\"高并发 & 秒杀 & 分布式\",\"collapsed\":true,\"items\":[{\"text\":\"分布式理论\",\"link\":\"/三高/分布式\"},{\"text\":\"分布式锁\",\"link\":\"/4、微服务/必备/分布式锁\"},{\"text\":\"秒杀\",\"link\":\"/三高/秒杀\"},{\"text\":\"高可用\",\"link\":\"/三高/高可用\"},{\"text\":\"高并发\",\"link\":\"/三高/高并发\"}]},{\"text\":\"云原生\",\"collapsed\":true,\"items\":[{\"text\":\"Docker\",\"link\":\"/云原生/Docker\"},{\"text\":\"K8S\",\"link\":\"/云原生/K8S\"}]},{\"text\":\"可视化 & 监控\",\"collapsed\":true,\"items\":[{\"text\":\"监控基础\",\"link\":\"/可视化 & 监控/监控基础\"},{\"text\":\"监控进阶\",\"link\":\"/可视化 & 监控/监控进阶\"},{\"text\":\"可视化大屏\",\"link\":\"/可视化 & 监控/可视化大屏\"},{\"text\":\"Zabbix\",\"link\":\"/可视化 & 监控/Zabbix\"}]},{\"text\":\"学前端\",\"collapsed\":true,\"items\":[{\"text\":\"HTML+CSS\",\"collapsed\":true,\"items\":[{\"text\":\"HTML基础\",\"link\":\"/1、学前端/1、HTML+CSS/HTML基础\"},{\"text\":\"CSS基础\",\"link\":\"/1、学前端/1、HTML+CSS/CSS基础\"},{\"text\":\"网页进阶\",\"link\":\"/1、学前端/1、HTML+CSS/网页进阶\"}]},{\"text\":\"JS+TS\",\"collapsed\":true,\"items\":[{\"text\":\"JS基础\",\"link\":\"/1、学前端/2、JS+TS/JS 基础\"},{\"text\":\"JS进阶\",\"link\":\"/1、学前端/2、JS+TS/JS 进阶\"},{\"text\":\"ES6基础\",\"link\":\"/1、学前端/2、JS+TS/ES6 基础\"},{\"text\":\"ES6进阶\",\"link\":\"/1、学前端/2、JS+TS/ES6 进阶\"},{\"text\":\"TS基础\",\"link\":\"/1、学前端/2、JS+TS/TypeScript\"}]},{\"text\":\"NodeJS\",\"collapsed\":true,\"items\":[{\"text\":\"Node基础\",\"link\":\"/1、学前端/4、Node/基础篇\"},{\"text\":\"Node进阶\",\"link\":\"/1、学前端/4、Node/进阶篇\"},{\"text\":\"项目实战\",\"link\":\"/1、学前端/4、Node/项目实战\"}]},{\"text\":\"Vue\",\"collapsed\":true,\"items\":[{\"text\":\"Vue3进阶\",\"link\":\"/1、学前端/3、Vue/Vue3/Vue3进阶\"},{\"text\":\"Vue3高级\",\"link\":\"/1、学前端/3、Vue/Vue3/Vue3高级\"},{\"text\":\"Vue3新语法\",\"link\":\"/1、学前端/3、Vue/Vue3/Vue3新语法\"},{\"text\":\"项目实战\",\"link\":\"/1、学前端/3、Vue/Vue2/Vue2项目\"}]},{\"text\":\"小程序\",\"collapsed\":true,\"items\":[{\"text\":\"小程序基础\",\"link\":\"/1、学前端/5、小程序/微信小程序\"},{\"text\":\"小程序优化\",\"link\":\"/1、学前端/5、小程序/小程序优化\"},{\"text\":\"uniapp\",\"link\":\"/1、学前端/5、小程序/uniapp\"},{\"text\":\"项目实战\",\"link\":\"/1、学前端/5、小程序/小程序项目\"}]}]},{\"text\":\"计算机基础\",\"collapsed\":true,\"items\":[{\"text\":\"数据结构\",\"link\":\"/计算机基础/数据结构/基础篇\"},{\"text\":\"操作系统\",\"link\":\"/计算机基础/计算机基础/操作系统\"},{\"text\":\"设计模式\",\"link\":\"/计算机基础/设计模式/基础篇\"},{\"text\":\"计算机网络\",\"link\":\"/计算机基础/计算机网络/网络基础\"},{\"text\":\"UML\",\"link\":\"/计算机基础/设计模式/UML\"},{\"text\":\"LeetCode\",\"link\":\"/计算机基础/算法/LeetCode\"}]},{\"text\":\"项目实战\",\"collapsed\":true,\"items\":[{\"text\":\"云尚办公\",\"collapsed\":true,\"items\":[{\"text\":\"基础篇\",\"link\":\"/项目实战/云尚办公/基础篇\"}]},{\"text\":\"小兔鲜\",\"collapsed\":true,\"items\":[{\"text\":\"基础篇\",\"link\":\"/项目实战/小兔鲜/基础篇\"},{\"text\":\"进阶篇1\",\"link\":\"/项目实战/小兔鲜/进阶篇1\"},{\"text\":\"进阶篇2\",\"link\":\"/项目实战/小兔鲜/进阶篇2\"}]},{\"text\":\"地图\",\"collapsed\":true,\"items\":[{\"text\":\"基础篇\",\"link\":\"/项目实战/百度地图/基础篇\"},{\"text\":\"进阶篇\",\"link\":\"/项目实战/百度地图/进阶篇\"}]},{\"text\":\"苍穹外卖\",\"collapsed\":true,\"items\":[{\"text\":\"进阶篇\",\"link\":\"/项目实战/苍穹外卖/进阶篇\"}]},{\"text\":\"黑马头条\",\"collapsed\":true,\"items\":[{\"text\":\"基础篇\",\"link\":\"/项目实战/黑马头条/基础篇\"},{\"text\":\"进阶篇\",\"link\":\"/项目实战/黑马头条/进阶篇\"},{\"text\":\"进阶篇2\",\"link\":\"/项目实战/黑马头条/进阶篇2\"},{\"text\":\"高级篇\",\"link\":\"/项目实战/黑马头条/高级篇\"}]},{\"text\":\"支付\",\"link\":\"/项目实战/支付\"},{\"text\":\"项目推荐\",\"link\":\"/项目实战/项目推荐\"}]},{\"text\":\"团队成员\",\"link\":\"/team\"}],\"siteTitle\":\"任硕的文档\",\"logo\":\"/Vue.png\",\"nav\":[{\"text\":\"Java学前端\",\"items\":[{\"items\":[{\"text\":\"HTML+JS\",\"link\":\"/Java学前端/HTML+JS\"},{\"text\":\"CSS\",\"link\":\"/Java学前端/CSS\"},{\"text\":\"Vue2+组件\",\"link\":\"/Java学前端/Vue2+组件\"},{\"text\":\"Vue3+组件\",\"link\":\"/Java学前端/Vue3+组件\"},{\"text\":\"React\",\"link\":\"/Java学前端/React\"}]}],\"activeMatch\":\"/Java/\"},{\"text\":\"软件测试\",\"items\":[{\"items\":[{\"text\":\"测试基础\",\"link\":\"/软件测试/测试基础\"},{\"text\":\"压力测试\",\"link\":\"/软件测试/压力测试\"}]}]},{\"text\":\"多线程\",\"items\":[{\"items\":[{\"text\":\"基础篇\",\"link\":\"/并发 & 多线程/基础篇\"},{\"text\":\"进阶篇\",\"link\":\"/并发 & 多线程/并发完善\"}]}]},{\"text\":\"开发工具\",\"items\":[{\"items\":[{\"text\":\"Chrome\",\"link\":\"/IDEA/Chrome\"},{\"text\":\"IDEA基础\",\"link\":\"/IDEA/IDEA基础\"},{\"text\":\"IDEA插件\",\"link\":\"/IDEA/IDEA插件\"},{\"text\":\"VS Code\",\"link\":\"/IDEA/VS Code\"}]}]},{\"text\":\"消息中间件\",\"items\":[{\"items\":[{\"text\":\"RabbitMQ\",\"link\":\"/消息中间件/RabbitMQ\"},{\"text\":\"RocketMQ\",\"link\":\"/消息中间件/RocketMQ\"},{\"text\":\"Kafka\",\"link\":\"/消息中间件/Kafka\"},{\"text\":\"Canal\",\"link\":\"/消息中间件/Canal\"}]}]}],\"socialLinks\":[{\"icon\":\"github\",\"link\":\"https://github.com/renshuo123/renshuo123.github.io\"},{\"icon\":\"twitter\",\"link\":\"#\"},{\"icon\":{\"svg\":\"<svg t=\\\"1676028692954\\\" class=\\\"icon\\\" ...</path></svg>\"},\"link\":\"https://github.com/\"}]},\"locales\":{},\"scrollOffset\":90,\"cleanUrls\":false}");</script>
    
  </body>
</html>